From fb4c5ab12a3a197378786e7135247890abcfe832 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 8 Oct 2026 12:45:28 +0200 Subject: [PATCH 01/91] feat: new CLI with `cdxt index` and `cdxt repackage` Replace the old `iter`/`size`/`warc` commands with two commands that can be piped: `cdxt index` selects captures from an index, `cdxt repackage` writes them to WARC files. cdxt index - indexes: CDX API (cc, ia, URLs), CDX files (local/remote, globs), Common Crawl's columnar URL index with DuckDB (cc-url-index-s3, cc-url-index-hf, *.parquet) or Athena (athena://...); detected from the INDEX argument or --index-format - selection: --url (CDX API patterns, matched as SURT keys for CDX files and SQL), --hostname, --domain, --url-file/--url-column/--url-file-type (CDX files), --crawl, --from/--to, --closest, --filter, --language (SQL), --url-fgrep(v), --limit, --sql (custom query), --count, --dry-run, cost confirmation (--yes) - output: -o (local or remote), JSONL, CDX/CDXJ, CSV, TSV, Parquet (optional pyarrow), --fields; one output per CDX file for a glob input and an -o path without extension (--processes) cdxt repackage - reads captures written by `cdxt index` (file, glob or stdin), copies the WARC records byte-for-byte with the async WARC fetcher from feat/warc-range-sources (http, s3 and hf:// reads; local, s3 and hf:// output; --processes, --parallel-readers, --add-metadata-record), --wb for wayback replays - multi-process workers share one file counter, so files get their final names (and warcinfo records) directly; the HF reader resolves each WARC file once - default --prefix REPACKAGE, human readable --size, no file without captures Other changes - `cdxt iter` is a deprecated alias of `cdxt index`; `size` and `warc` exit with the equivalent new command line - fix pad_timestamp_up() for timestamps with a day and for leap years - remove the deprecated cdx_iter and cdx_size scripts - new extras: athena, duckdb, hf, parquet; fsspec, aiofiles and url-is-in are required --- cdx_toolkit/cli.py | 245 ------- cdx_toolkit/cli/__init__.py | 42 ++ cdx_toolkit/cli/__main__.py | 3 + cdx_toolkit/cli/common.py | 55 ++ cdx_toolkit/cli/deprecated.py | 108 ++++ cdx_toolkit/cli/index.py | 251 ++++++++ cdx_toolkit/cli/repackage.py | 241 +++++++ cdx_toolkit/indexes/__init__.py | 3 + cdx_toolkit/indexes/base.py | 57 ++ cdx_toolkit/indexes/cdx_api.py | 90 +++ cdx_toolkit/indexes/cdx_files.py | 254 ++++++++ cdx_toolkit/indexes/patterns.py | 58 ++ cdx_toolkit/indexes/query.py | 55 ++ cdx_toolkit/indexes/resolve.py | 81 +++ cdx_toolkit/indexes/sql.py | 313 +++++++++ cdx_toolkit/record_io.py | 281 ++++++++ cdx_toolkit/repackage/__init__.py | 0 cdx_toolkit/repackage/data_classes.py | 121 ++++ cdx_toolkit/repackage/hf_utils.py | 262 ++++++++ cdx_toolkit/repackage/local_writer.py | 37 ++ cdx_toolkit/repackage/multiprocess.py | 206 ++++++ cdx_toolkit/repackage/s3_utils.py | 62 ++ cdx_toolkit/repackage/s3_writer.py | 172 +++++ cdx_toolkit/repackage/sources/__init__.py | 3 + cdx_toolkit/repackage/sources/athena.py | 202 ++++++ cdx_toolkit/repackage/sources/base.py | 35 + cdx_toolkit/repackage/sources/csv.py | 120 ++++ cdx_toolkit/repackage/sources/duckdb.py | 139 ++++ cdx_toolkit/repackage/sources/records.py | 45 ++ cdx_toolkit/repackage/sources/sql_base.py | 199 ++++++ cdx_toolkit/repackage/warc_filter.py | 599 ++++++++++++++++++ cdx_toolkit/repackage/warc_utils.py | 154 +++++ cdx_toolkit/timeutils.py | 17 +- cdx_toolkit/utils.py | 45 -- requirements.txt | 18 + scripts/cdx_iter | 151 ----- scripts/cdx_size | 31 - setup.py | 27 +- tests/conftest.py | 154 ++++- .../test_cli/test_iter_alias.jsonl | 1 + .../test_multi_misc_not_ia_slow_5.jsonl | 1 - .../filtered_CC-MAIN-2024-30_cdx-00187.gz | Bin 0 -> 63684 bytes ...ed_CC-MAIN-2024-30_cdx-00187.metadata.json | 4 + tests/data/repackage/url_list.txt | 10 + tests/repackage/__init__.py | 0 tests/repackage/test_athena_prompt.py | 63 ++ tests/repackage/test_athena_query_builder.py | 223 +++++++ tests/repackage/test_csv_source.py | 130 ++++ tests/repackage/test_data_classes.py | 12 + tests/repackage/test_hf_utils.py | 188 ++++++ tests/repackage/test_local_writer.py | 474 ++++++++++++++ tests/repackage/test_multiprocess_naming.py | 54 ++ tests/repackage/test_producer.py | 50 ++ tests/repackage/test_repackage_cli.py | 205 ++++++ tests/repackage/test_s3_utils.py | 180 ++++++ tests/repackage/test_s3_writer.py | 311 +++++++++ tests/repackage/test_sql_index_gated.py | 81 +++ tests/repackage/test_warc_filter.py | 432 +++++++++++++ tests/repackage/test_warc_from_fs.py | 53 ++ tests/repackage/test_warc_utils.py | 14 + tests/repackage/test_warc_writer.py | 142 +++++ tests/test_cli.py | 401 +++++------- tests/unit/test_cc.py | 2 +- tests/unit/test_cdx_files.py | 193 ++++++ tests/unit/test_cli.py | 272 ++++++-- tests/unit/test_indexes.py | 131 ++++ tests/unit/test_record_io.py | 146 +++++ tests/unit/test_sql_indexes.py | 305 +++++++++ tests/unit/test_timeutils.py | 3 + 69 files changed, 8190 insertions(+), 827 deletions(-) delete mode 100644 cdx_toolkit/cli.py create mode 100644 cdx_toolkit/cli/__init__.py create mode 100644 cdx_toolkit/cli/__main__.py create mode 100644 cdx_toolkit/cli/common.py create mode 100644 cdx_toolkit/cli/deprecated.py create mode 100644 cdx_toolkit/cli/index.py create mode 100644 cdx_toolkit/cli/repackage.py create mode 100644 cdx_toolkit/indexes/__init__.py create mode 100644 cdx_toolkit/indexes/base.py create mode 100644 cdx_toolkit/indexes/cdx_api.py create mode 100644 cdx_toolkit/indexes/cdx_files.py create mode 100644 cdx_toolkit/indexes/patterns.py create mode 100644 cdx_toolkit/indexes/query.py create mode 100644 cdx_toolkit/indexes/resolve.py create mode 100644 cdx_toolkit/indexes/sql.py create mode 100644 cdx_toolkit/record_io.py create mode 100644 cdx_toolkit/repackage/__init__.py create mode 100644 cdx_toolkit/repackage/data_classes.py create mode 100644 cdx_toolkit/repackage/hf_utils.py create mode 100644 cdx_toolkit/repackage/local_writer.py create mode 100644 cdx_toolkit/repackage/multiprocess.py create mode 100644 cdx_toolkit/repackage/s3_utils.py create mode 100644 cdx_toolkit/repackage/s3_writer.py create mode 100644 cdx_toolkit/repackage/sources/__init__.py create mode 100644 cdx_toolkit/repackage/sources/athena.py create mode 100644 cdx_toolkit/repackage/sources/base.py create mode 100644 cdx_toolkit/repackage/sources/csv.py create mode 100644 cdx_toolkit/repackage/sources/duckdb.py create mode 100644 cdx_toolkit/repackage/sources/records.py create mode 100644 cdx_toolkit/repackage/sources/sql_base.py create mode 100644 cdx_toolkit/repackage/warc_filter.py create mode 100644 cdx_toolkit/repackage/warc_utils.py delete mode 100644 scripts/cdx_iter delete mode 100644 scripts/cdx_size create mode 100644 tests/data/mock_responses/test_cli/test_iter_alias.jsonl delete mode 100644 tests/data/mock_responses/test_cli/test_multi_misc_not_ia_slow_5.jsonl create mode 100644 tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.gz create mode 100644 tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.metadata.json create mode 100644 tests/data/repackage/url_list.txt create mode 100644 tests/repackage/__init__.py create mode 100644 tests/repackage/test_athena_prompt.py create mode 100644 tests/repackage/test_athena_query_builder.py create mode 100644 tests/repackage/test_csv_source.py create mode 100644 tests/repackage/test_data_classes.py create mode 100644 tests/repackage/test_hf_utils.py create mode 100644 tests/repackage/test_local_writer.py create mode 100644 tests/repackage/test_multiprocess_naming.py create mode 100644 tests/repackage/test_producer.py create mode 100644 tests/repackage/test_repackage_cli.py create mode 100644 tests/repackage/test_s3_utils.py create mode 100644 tests/repackage/test_s3_writer.py create mode 100644 tests/repackage/test_sql_index_gated.py create mode 100644 tests/repackage/test_warc_filter.py create mode 100644 tests/repackage/test_warc_from_fs.py create mode 100644 tests/repackage/test_warc_utils.py create mode 100644 tests/repackage/test_warc_writer.py create mode 100644 tests/unit/test_cdx_files.py create mode 100644 tests/unit/test_indexes.py create mode 100644 tests/unit/test_record_io.py create mode 100644 tests/unit/test_sql_indexes.py diff --git a/cdx_toolkit/cli.py b/cdx_toolkit/cli.py deleted file mode 100644 index 1e59e04..0000000 --- a/cdx_toolkit/cli.py +++ /dev/null @@ -1,245 +0,0 @@ -from argparse import ArgumentParser, Namespace -import logging -import csv -import sys -import json -import os - -import cdx_toolkit - -from cdx_toolkit.utils import get_version, setup_cdx_fetcher_and_kwargs - - -LOGGER = logging.getLogger(__name__) - - -def main(args=None): - parser = ArgumentParser(description='cdx_toolkit iterator command line tool') - - parser.add_argument('--version', '-V', action='version', version=get_version()) - parser.add_argument( - '--verbose', - '-v', - action='count', - help='set logging level to INFO (-v) or DEBUG (-vv)' - ) - parser.add_argument( - '--cc', - action='store_const', - const='cc', - help='direct the query to the Common Crawl CDX/WARCs' - ) - parser.add_argument( - '--crawl', - action='store', - help=('crawl names (comma separated) or an integer for the most recent N crawls. ' - 'Implies --cc') - ) - parser.add_argument( - '--ia', - action='store_const', - const='ia', - help='direct the query to the Internet Archive CDX/wayback' - ) - parser.add_argument('--source', action='store', help='direct the query to this CDX server') - parser.add_argument('--wb', action='store', help='direct replays for content to this wayback') - parser.add_argument('--limit', type=int, action='store') - parser.add_argument('--cc-mirror', action='store', help='use this Common Crawl index mirror') - parser.add_argument('--cc-sort', action='store', help='default mixed, alternatively: ascending') - parser.add_argument('--from', action='store') - parser.add_argument('--to', action='store') - parser.add_argument('--filter', action='append', help='see CDX API documentation for usage') - parser.add_argument( - '--get', - action='store_true', - help='use a single get instead of a paged iteration. default limit=1000' - ) - parser.add_argument( - '--closest', - action='store', - help='get the closest capture to this timestamp. use with --get' - ) - - subparsers = parser.add_subparsers(dest='cmd') - subparsers.required = True - - iterate = subparsers.add_parser('iter', help='iterate printing captures') - iterate.add_argument('--all-fields', action='store_true') - iterate.add_argument( - '--fields', - action='store', - default='url,status,timestamp', - help='try --all-fields if you need the list' - ) - iterate.add_argument('--jsonl', action='store_true') - iterate.add_argument('--csv', action='store_true') - iterate.add_argument('url') - iterate.set_defaults(func=iterator) - - warc = subparsers.add_parser('warc', help='iterate over capture content, creating a warc') - warc.add_argument('--prefix', default='TEST', help='prefix for the warc filename') - warc.add_argument( - '--subprefix', - type=str, - default=None, - help='subprefix for the warc filename, default None' - ) - warc.add_argument( - '--size', - type=int, - default=1000000000, - help='target for the warc filesize in bytes' - ) - warc.add_argument( - '--creator', - action='store', - help='creator of the warc: person, organization, service' - ) - warc.add_argument( - '--operator', - action='store', - help='a person, if the creator is an organization' - ) - warc.add_argument( - '--url-fgrep', - action='store', - help='this pattern must be present to warc an url' - ) - warc.add_argument( - '--url-fgrepv', - action='store', - help='this pattern must not be present to warc an url, e.g. /robots.txt' - ) - warc.add_argument( - '--warc-download-prefix', - action='store', - help='prefix for downloading content, automatically set for CC' - ) - warc.add_argument('url') - warc.set_defaults(func=warcer) - - size = subparsers.add_parser('size', help='imprecise count of how many results are available') - size.add_argument('--details', action='store_true', help='show details of each subindex') - size.add_argument('url') - size.set_defaults(func=sizer) - - if args is not None: - cmdline = ' '.join(args) - else: # pragma: no cover - # there's something magic about args and console_scripts - # this fallback is needed when installed by setuptools - if len(sys.argv) > 1: - cmdline = 'cdxt ' + ' '.join(sys.argv[1:]) - else: - cmdline = 'cdxt' - cmd = parser.parse_args(args=args) - set_loglevel(cmd) - cmd.func(cmd, cmdline) - - -def set_loglevel(cmd: Namespace): - loglevel = os.getenv('LOGLEVEL') or 'WARNING' - if cmd.verbose: - if cmd.verbose > 0: - loglevel = 'INFO' - if cmd.verbose > 1: - loglevel = 'DEBUG' - - # because pytest has already initialized the logger, we have to set the root logger - logging.getLogger().setLevel(loglevel) - # for the normal case - logging.basicConfig(level=loglevel) - - LOGGER.info('set loglevel to %s', str(loglevel)) - - -def winnow_fields(cmd: Namespace, fields, obj): - if cmd.all_fields: - printme = obj - else: - printme = dict([(k, obj[k]) for k in fields if k in obj]) - return printme - - -def print_line(cmd: Namespace, writer, printme): - if cmd.jsonl: - print(json.dumps(printme, sort_keys=True)) - elif writer: - writer.writerow(printme) - else: - print(', '.join([' '.join((k, printme[k])) for k in sorted(printme.keys())])) - - -def iterator(cmd: Namespace, cmdline): - cdx, kwargs = setup_cdx_fetcher_and_kwargs(cmd) - fields = set(cmd.fields.split(',')) - if cmd.csv: - writer = csv.DictWriter(sys.stdout, fieldnames=sorted(list(fields))) - writer.writeheader() - else: - writer = None - - if cmd.get: - objs = cdx.get(cmd.url, **kwargs) - for obj in objs: - printme = winnow_fields(cmd, fields, obj) - print_line(cmd, writer, printme) - return - - for obj in cdx.iter(cmd.url, **kwargs): - printme = winnow_fields(cmd, fields, obj) - print_line(cmd, writer, printme) - - -def warcer(cmd: Namespace, cmdline: str): - cdx, kwargs = setup_cdx_fetcher_and_kwargs(cmd) - - ispartof = cmd.prefix - if cmd.subprefix: - ispartof += '-' + cmd.subprefix - - info = { - 'software': 'pypi_cdx_toolkit/'+get_version(), - 'isPartOf': ispartof, - 'description': 'warc extraction generated with: '+cmdline, - 'format': 'WARC file version 1.0', # todo: if we directly read a warc, have this match the warc - } - if cmd.creator: - info['creator'] = cmd.creator - if cmd.operator: - info['operator'] = cmd.operator - - kwargs_writer = {} - if 'size' in kwargs: - kwargs_writer['size'] = kwargs['size'] - del kwargs['size'] - - with cdx_toolkit.warc.get_writer(cmd.prefix, cmd.subprefix, info, **kwargs_writer) as writer: - for obj in cdx.iter(cmd.url, **kwargs): - url = obj['url'] - if cmd.url_fgrep and cmd.url_fgrep not in url: - LOGGER.debug('not warcing due to fgrep: %s', url) - continue - if cmd.url_fgrepv and cmd.url_fgrepv in url: - LOGGER.debug('not warcing due to fgrepv: %s', url) - continue - timestamp = obj['timestamp'] - try: - record = obj.fetch_warc_record() - except RuntimeError: # pragma: no cover - LOGGER.warning('skipping capture for RuntimeError 404: %s %s', url, timestamp) - continue - if obj.is_revisit(): - LOGGER.warning('revisit record being resolved for url %s %s', url, timestamp) - writer.write_record(record) - - -def sizer(cmd: Namespace, cmdline): - cdx, kwargs = setup_cdx_fetcher_and_kwargs(cmd) - - size = cdx.get_size_estimate(cmd.url, **kwargs) - print(size) - - -if __name__ == "__main__": - main() diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py new file mode 100644 index 0000000..85863a2 --- /dev/null +++ b/cdx_toolkit/cli/__init__.py @@ -0,0 +1,42 @@ +from argparse import ArgumentParser +import sys + +from cdx_toolkit.indexes import UsageError +from cdx_toolkit.utils import get_version + +from . import index, repackage +from .common import add_verbose_arg, set_loglevel +from .deprecated import DEPRECATED_COMMANDS, REMOVED_COMMANDS, exit_removed_command, get_command +from .deprecated import rewrite_deprecated_command + + +def build_parser() -> ArgumentParser: + parser = ArgumentParser(prog='cdxt', description='cdx_toolkit command line tool') + parser.add_argument('--version', '-V', action='version', version=get_version()) + add_verbose_arg(parser) + + subparsers = parser.add_subparsers(dest='cmd', metavar='COMMAND') + subparsers.required = True + index.add_parser(subparsers) + repackage.add_parser(subparsers) + return parser + + +def main(args=None): + if args is None: # pragma: no cover + args = sys.argv[1:] + + command = get_command(args) + if command in REMOVED_COMMANDS: + exit_removed_command(command, args) + if command in DEPRECATED_COMMANDS: + args = rewrite_deprecated_command(args) + + parser = build_parser() + cmd = parser.parse_args(args=args) + set_loglevel(cmd.verbose) + cmdline = ' '.join(['cdxt'] + list(args)) + try: + cmd.func(cmd, cmdline) + except (UsageError, FileNotFoundError) as e: + parser.exit(2, f'cdxt {cmd.cmd}: error: {e}\n') diff --git a/cdx_toolkit/cli/__main__.py b/cdx_toolkit/cli/__main__.py new file mode 100644 index 0000000..8273c4f --- /dev/null +++ b/cdx_toolkit/cli/__main__.py @@ -0,0 +1,3 @@ +from . import main + +main() diff --git a/cdx_toolkit/cli/common.py b/cdx_toolkit/cli/common.py new file mode 100644 index 0000000..9052f19 --- /dev/null +++ b/cdx_toolkit/cli/common.py @@ -0,0 +1,55 @@ +from argparse import SUPPRESS, ArgumentParser, ArgumentTypeError +import logging +import os +import re + + +LOGGER = logging.getLogger(__name__) + +SIZE_UNITS = { + '': 1, + 'b': 1, + 'k': 1000, 'kb': 1000, 'kib': 1024, + 'm': 1000 ** 2, 'mb': 1000 ** 2, 'mib': 1024 ** 2, + 'g': 1000 ** 3, 'gb': 1000 ** 3, 'gib': 1024 ** 3, + 't': 1000 ** 4, 'tb': 1000 ** 4, 'tib': 1024 ** 4, +} + + +def add_verbose_arg(parser: ArgumentParser, subcommand: bool = False): + # On subcommands the default is suppressed, so it does not override a -v given before the subcommand. + parser.add_argument( + '--verbose', + '-v', + action='count', + default=SUPPRESS if subcommand else 0, + help='set logging level to INFO (-v) or DEBUG (-vv)', + ) + + +def set_loglevel(verbose: int): + loglevel = os.getenv('LOGLEVEL') or 'WARNING' + if verbose and verbose > 0: + loglevel = 'INFO' + if verbose and verbose > 1: + loglevel = 'DEBUG' + + # because pytest has already initialized the logger, we have to set the root logger + logging.getLogger().setLevel(loglevel) + # for the normal case + logging.basicConfig(level=loglevel) + + LOGGER.info('set loglevel to %s', str(loglevel)) + + +def parse_size(value: str) -> int: + """Parse a size in bytes like 1000000, 100MB, 1.5GB or 1GiB (argparse type).""" + match = re.fullmatch(r'\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]*)\s*', value) + if not match or match.group(2).lower() not in SIZE_UNITS: + raise ArgumentTypeError(f'invalid size {value!r}, use e.g. 1000000, 100MB or 1GiB') + return int(float(match.group(1)) * SIZE_UNITS[match.group(2).lower()]) + + +def parse_fields(value: str): + """Comma-separated list of field names (argparse type).""" + return [f.strip() for f in value.split(',') if f.strip()] diff --git a/cdx_toolkit/cli/deprecated.py b/cdx_toolkit/cli/deprecated.py new file mode 100644 index 0000000..3633e5d --- /dev/null +++ b/cdx_toolkit/cli/deprecated.py @@ -0,0 +1,108 @@ +''' +Old commands: `iter` is a deprecated alias of `index` (same arguments); `size` and `warc` were removed and +print the equivalent new command line. +''' +import shlex +import sys +from typing import List, Optional, Tuple + + +DEPRECATED_COMMANDS = {'iter': 'index'} +REMOVED_COMMANDS = {'size': 'index --count', 'warc': 'index | cdxt repackage'} + + +DOCS_URL = 'https://github.com/commoncrawl/cdx_toolkit#command-line-tools' + +# options of the old command line that take a value +VALUE_OPTIONS = { + '--crawl', '--source', '--wb', '--limit', '--cc-mirror', '--cc-sort', '--from', '--to', '--filter', + '--closest', '--fields', '--prefix', '--subprefix', '--size', '--creator', '--operator', '--url-fgrep', + '--url-fgrepv', '--warc-download-prefix', +} +INDEX_OPTIONS = { + '--crawl', '--limit', '--cc-mirror', '--cc-sort', '--from', '--to', '--filter', '--closest', '--fields', + '--url-fgrep', '--url-fgrepv', +} +REPACKAGE_OPTIONS = {'--wb', '--prefix', '--subprefix', '--size', '--creator', '--operator', '--warc-download-prefix'} + + +def parse_old_args(args: List[str]) -> Tuple[List[Tuple[str, Optional[str]]], List[Tuple[int, str]]]: + """Split a command line into (option, value) pairs and (position, value) of the positional arguments.""" + options, positionals = [], [] + tokens = iter(enumerate(args)) + for i, token in tokens: + if token.startswith('-') and token != '-': + if '=' in token: + options.append(tuple(token.split('=', 1))) + elif token in VALUE_OPTIONS: + options.append((token, next(tokens, (None, ''))[1])) + else: + options.append((token, None)) + else: + positionals.append((i, token)) + return options, positionals + + +def get_command(args: List[str]) -> Optional[str]: + _, positionals = parse_old_args(args) + return positionals[0][1] if positionals else None + + +def translate(args: List[str]) -> str: + """Best-effort translation of an old size/warc command line to the new syntax.""" + options, positionals = parse_old_args(args) + command, urls = positionals[0][1], [url for _, url in positionals[1:]] + + index, index_args, repackage_args = None, [], [] + for option, value in options: + if option in ('--cc', '--ia'): + index = option[2:] + elif option == '--source': + index = value + elif option in ('-v', '-vv', '--verbose'): + index_args.append(option) + elif option == '--jsonl': + index_args += ['--output-format', 'jsonl'] + elif option == '--csv': + index_args += ['--output-format', 'csv'] + elif option in INDEX_OPTIONS: + index_args += [option, value] + elif option in REPACKAGE_OPTIONS: + repackage_args += [option, value] + # --get, --all-fields and --details have no equivalent (all fields are the default) + + if index is None: + index = 'cc' if any(option == '--crawl' for option, _ in options) else 'INDEX' + url_args = [arg for url in urls for arg in ('--url', url)] + + line = ['cdxt', 'index', index] + url_args + index_args + if command == 'size': + line.append('--count') + elif command == 'warc': + if index == 'ia' and '--wb' not in repackage_args: + repackage_args += ['--wb', 'ia'] + line += ['|', 'cdxt', 'repackage'] + repackage_args + return ' '.join(token if token == '|' else shlex.quote(token) for token in line) + + +def rewrite_deprecated_command(args: List[str]) -> List[str]: + """Run `cdxt iter ...` exactly like `cdxt index ...` (same arguments), with a deprecation warning on stderr.""" + _, positionals = parse_old_args(args) + position, command = positionals[0] + new_command = DEPRECATED_COMMANDS[command] + sys.stderr.write( + f'cdxt: "{command}" is deprecated and will be removed in a future version, use "cdxt {new_command}" ' + f'(same arguments). See {DOCS_URL}\n' + ) + return args[:position] + [new_command] + args[position + 1:] + + +def exit_removed_command(command: str, args: List[str]): + message = f'cdxt: the "{command}" command has been removed, use "cdxt {REMOVED_COMMANDS[command]}" instead.\n' + try: + message += f'\nThe equivalent command is:\n\n {translate(args)}\n' + except Exception: # pragma: no cover - the translation is only a hint + pass + message += f'\nSee "cdxt index --help", "cdxt repackage --help" or {DOCS_URL}\n' + sys.stderr.write(message) + raise SystemExit(2) diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py new file mode 100644 index 0000000..b11186f --- /dev/null +++ b/cdx_toolkit/cli/index.py @@ -0,0 +1,251 @@ +from argparse import Namespace, RawDescriptionHelpFormatter +import logging + +from cdx_toolkit.indexes import IndexQuery, UsageError, iter_index_records, resolve_index +from cdx_toolkit.indexes.resolve import INDEX_FORMATS +from cdx_toolkit.record_io import ( + FORMAT_CHOICES, canonical_format, format_from_path, import_pyarrow, open_text, write_records, +) + +from .common import add_verbose_arg, parse_fields + + +LOGGER = logging.getLogger(__name__) + +DESCRIPTION = '''\ +Select captures from an index and write them as JSONL, CDXJ, CSV, TSV or Parquet. + +INDEX is one of: + cc, ia Common Crawl or Internet Archive CDX API + http(s)://... URL of a CDX API server + cc-url-index-s3 Common Crawl's columnar URL index on S3, queried with DuckDB + cc-url-index-hf Common Crawl's columnar URL index on Hugging Face, queried with DuckDB + PATH other columnar (parquet) index: *.parquet files, or a hive-partitioned + directory with --index-format duckdb + athena://... the columnar URL index queried with AWS Athena, e.g. + athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://bucket/path/ + PATH CDX file(s), local or remote, may be a glob, e.g. + 's3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz' +''' + +EXAMPLES = '''\ +examples: + cdxt index cc --url 'commoncrawl.org/*' --limit 10 + cdxt index cc --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.csv + cdxt index ia --url 'commoncrawl.org/*' --from 2008 --to 200906302359 -o captures.jsonl + cdxt index cc --url 'commoncrawl.org/*' --count + cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.jsonl + cdxt index cc-url-index-s3 --sql homepages.sql --yes -o captures.csv + cdxt index 'indexes/cdx-*.gz' --url-file urls.txt -o filtered/ # one output per CDX file + cdxt index cc --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --prefix COMMONCRAWL +''' + + +def add_parser(subparsers): + parser = subparsers.add_parser( + 'index', + help='select captures from an index', + description=DESCRIPTION, + epilog=EXAMPLES, + formatter_class=RawDescriptionHelpFormatter, + ) + add_verbose_arg(parser, subcommand=True) + parser.add_argument('index', metavar='INDEX', help='index name or location (see above)') + parser.add_argument( + '--index-format', + choices=INDEX_FORMATS, + default='auto', + help='type of the index (default: detected from INDEX)', + ) + + selection = parser.add_argument_group( + 'capture selection', + ('--url, --hostname, --domain and --url-file can be repeated and combined; a capture matches if it matches ' + 'any of them.'), + ) + selection.add_argument( + '--url', action='append', dest='urls', default=[], metavar='PATTERN', + help="URL pattern, e.g. 'example.com/*' (prefix), '*.example.com' (domain) or an exact URL", + ) + selection.add_argument( + '--hostname', action='append', dest='hostnames', default=[], metavar='HOST', + help='all captures of this exact hostname, e.g. www.example.com', + ) + selection.add_argument( + '--domain', action='append', dest='domains', default=[], metavar='DOMAIN', + help='all captures of this domain and its subdomains, e.g. example.com', + ) + selection.add_argument( + '--url-file', metavar='PATH', + help=('file with URL patterns (like --url): one per line, or a column of a CSV, TSV, JSONL or Parquet file; ' + 'CDX files only'), + ) + selection.add_argument( + '--url-column', metavar='NAME', help='column of the --url-file with the URL patterns (default: url)' + ) + selection.add_argument( + '--url-file-type', choices=('pattern', 'domain'), + help=('how the --url-file entries match: pattern (default) like --url, e.g. example.com is only that URL; ' + 'domain: example.com matches the domain and its subdomains, example.com/path all URLs starting with it'), + ) + selection.add_argument( + '--crawl', + help=('Common Crawl crawl names (comma separated, e.g. CC-MAIN-2024-33 or CC-MAIN-2018) ' + 'or an integer for the most recent N crawls'), + ) + selection.add_argument('--from', dest='from_ts', metavar='TIMESTAMP', help='start time, e.g. 2008 or 20080328') + selection.add_argument('--to', metavar='TIMESTAMP', help='end time, e.g. 2009 or 200906302359') + selection.add_argument('--closest', metavar='TIMESTAMP', help='captures closest to this timestamp (CDX API only)') + selection.add_argument( + '--filter', action='append', dest='filters', default=[], metavar='[!~=]FIELD:EXPR', + help='CDX API filter, e.g. =status:200 (repeatable; see README for the syntax)', + ) + selection.add_argument( + '--language', action='append', dest='languages', default=[], metavar='CODE', + help=('only captures whose primary content language is CODE (ISO 639-3, e.g. eng or deu; repeatable); ' + 'SQL indexes only'), + ) + selection.add_argument('--url-fgrep', metavar='TEXT', help='only captures whose URL contains TEXT') + selection.add_argument('--url-fgrepv', metavar='TEXT', help='only captures whose URL does not contain TEXT') + selection.add_argument( + '--limit', type=int, + help='maximum number of captures to read from the index (applied before --url-fgrep/--url-fgrepv)', + ) + selection.add_argument( + '--sql', metavar='PATH', + help=("file with a SQL query for Athena or DuckDB ('-' = stdin), used instead of " + '--hostname/--domain/--crawl/--from/--to/--language/--limit'), + ) + + output = parser.add_argument_group('output') + output.add_argument( + '-o', '--output', default='-', metavar='PATH', + help=("output file, local or remote (e.g. s3://...), gzipped if it ends with .gz (default: '-' = stdout); " + 'for CDX files, a path without file extension is a directory with one output per CDX file'), + ) + output.add_argument( + '--output-format', choices=FORMAT_CHOICES, + help=('output format (default: detected from the --output extension, jsonl for stdout); cdx is the same as ' + "cdxj: Common Crawl's index files have the .cdx extension but use the CDXJ format (crawl index JSON); " + 'parquet needs cdx_toolkit[parquet]'), + ) + output.add_argument( + '--fields', type=parse_fields, metavar='FIELD,...', + help='comma-separated fields to output, e.g. url,timestamp,status (default: all fields)', + ) + output.add_argument( + '--count', action='store_true', + help=('only print the number of captures (CDX API: cheap estimate based on the number of index pages; ' + 'SQL: exact count)'), + ) + output.add_argument( + '--dry-run', action='store_true', + help='only print what would be queried (e.g. the SQL query), without querying the index', + ) + + cdx_files = parser.add_argument_group('CDX files options') + cdx_files.add_argument( + '--processes', type=int, default=1, metavar='N', + help='number of processes, with one output per CDX file (default: 1)', + ) + + sql = parser.add_argument_group('SQL options (Athena, DuckDB)') + sql.add_argument( + '--yes', action='store_true', + help=('do not ask for confirmation before scanning more than 10 crawls ' + '(Athena bills per TB scanned; restrict with --crawl)'), + ) + + cdx_api = parser.add_argument_group('CDX API options') + cdx_api.add_argument('--cc-mirror', metavar='URL', help='use this Common Crawl index mirror') + cdx_api.add_argument('--cc-sort', choices=('mixed', 'ascending'), help='order of the crawls (default: mixed)') + + parser.set_defaults(func=run) + return parser + + +def read_sql(path: str) -> str: + with open_text(path, 'r') as fh: + return fh.read() + + +def build_query(args: Namespace) -> IndexQuery: + return IndexQuery( + urls=args.urls, + hostnames=args.hostnames, + domains=args.domains, + url_file=args.url_file, + url_column=args.url_column, + url_file_type=args.url_file_type, + crawl=args.crawl, + from_ts=args.from_ts, + to=args.to, + closest=args.closest, + filters=args.filters, + url_fgrep=args.url_fgrep, + url_fgrepv=args.url_fgrepv, + limit=args.limit, + languages=args.languages, + sql=read_sql(args.sql) if args.sql else None, + ) + + +def get_output_format(args: Namespace) -> str: + fmt = canonical_format(args.output_format) + if not fmt: + fmt = 'jsonl' if args.output == '-' else format_from_path(args.output) + if fmt is None: + raise UsageError(f'cannot detect the output format of {args.output}, please set --output-format') + if fmt == 'parquet': + if args.output == '-': + raise UsageError('Parquet output cannot be written to stdout, please set --output') + try: + import_pyarrow() + except RuntimeError as e: + raise UsageError(str(e)) from None + return fmt + + +def writes_per_file(args: Namespace, backend) -> bool: + """CDX files with an output path without file extension: one output per CDX file.""" + from cdx_toolkit.indexes.cdx_files import CdxFilesBackend + + return ( + isinstance(backend, CdxFilesBackend) + and not (args.count or args.dry_run or args.output_format) + and args.output != '-' + and format_from_path(args.output) is None + ) + + +def run(args: Namespace, cmdline: str): + backend = resolve_index( + args.index, args.index_format, cc_mirror=args.cc_mirror, cc_sort=args.cc_sort, assume_yes=args.yes + ) + query = build_query(args) + backend.validate(query) + + if writes_per_file(args, backend): + if args.fields: + raise UsageError('--fields cannot be used when writing one output per CDX file (the lines are copied)') + n_files, n = backend.write_per_file(query, args.output, processes=args.processes) + LOGGER.info('wrote %d captures to %d files in %s', n, n_files, args.output) + return + if args.processes > 1: + raise UsageError('--processes can only be used with CDX files and one output per file (-o DIRECTORY)') + + output_format = None if args.count or args.dry_run else get_output_format(args) + + if args.dry_run: + with open_text(args.output, 'w') as fh: + fh.write(backend.explain(query)) + return + + if args.count: + count = backend.count(query) + with open_text(args.output, 'w') as fh: + fh.write(f'{count}\n') + return + + n = write_records(iter_index_records(backend, query), args.output, output_format, args.fields) + LOGGER.info('wrote %d captures to %s', n, args.output) diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py new file mode 100644 index 0000000..e7a4446 --- /dev/null +++ b/cdx_toolkit/cli/repackage.py @@ -0,0 +1,241 @@ +from argparse import Namespace, RawDescriptionHelpFormatter +import logging +import os +import time +from typing import Mapping, Optional + +import cdx_toolkit.warc +from cdx_toolkit.indexes import UsageError +from cdx_toolkit.record_io import ( + FORMAT_CHOICES, canonical_format, check_glob, format_from_path, import_pyarrow, read_records, +) +from cdx_toolkit.utils import get_version +from cdx_toolkit.warc import fetch_wb_warc + +from .common import add_verbose_arg, parse_size + + +LOGGER = logging.getLogger(__name__) + +# wayback names that map to replay URLs +WAYBACK_NAMES = {'ia': 'https://web.archive.org/web'} + +DESCRIPTION = '''\ +Fetch the captures listed in an index file (as written by `cdxt index`) and write them to WARC files. + +Captures with a WARC location (filename, offset, length; e.g. Common Crawl captures) are copied +byte-for-byte from --warc-download-prefix (https://, s3:// or hf://buckets/...) into +[-]-001.warc.gz, -002, ... (local, s3:// or hf://buckets/...). + +Captures without a WARC location (e.g. Internet Archive captures) are replayed from a wayback +machine with --wb and written to [-]-000000.extracted.warc.gz, ... +''' + +EXAMPLES = '''\ +examples: + cdxt index cc --url 'commoncrawl.org/*' --limit 10 -o captures.jsonl + cdxt repackage captures.jsonl --prefix COMMONCRAWL --size 100MB + + cdxt index cc-url-index-s3 --sql homepages.sql --yes -o homepages.csv + cdxt repackage homepages.csv --prefix s3://my-bucket/HOMEPAGES --warc-download-prefix s3://commoncrawl \\ + --processes 8 --parallel-readers 16 + + cdxt index ia --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --wb ia --prefix IA +''' + + +def add_parser(subparsers): + parser = subparsers.add_parser( + 'repackage', + help='write the captures of an index file to WARC files', + description=DESCRIPTION, + epilog=EXAMPLES, + formatter_class=RawDescriptionHelpFormatter, + ) + add_verbose_arg(parser, subcommand=True) + parser.add_argument( + 'input', nargs='?', default='-', metavar='INDEX_PATH', + help="index file(s) with captures, local or remote, may be a glob (default: '-' = stdin)", + ) + parser.add_argument( + '--index-format', choices=('auto',) + FORMAT_CHOICES, default='auto', + help=('format of the index file (default: parquet for *.parquet, otherwise detected from the first line); ' + "cdx is the same as cdxj: Common Crawl's index files have the .cdx extension but use the CDXJ format " + '(crawl index JSON)'), + ) + + source = parser.add_argument_group('fetching') + source.add_argument( + '--warc-download-prefix', default='https://data.commoncrawl.org', metavar='URL', + help=('prefix for downloading WARC records by filename (default: %(default)s); s3://commoncrawl is ' + 'fastest in AWS us-east-1, hf://buckets/commoncrawl/commoncrawl reads from Hugging Face'), + ) + source.add_argument( + '--wb', metavar='URL', + help="fetch captures by replaying them from this wayback machine ('ia' = Internet Archive)", + ) + source.add_argument( + '--processes', type=int, default=1, + help='number of worker processes, e.g. the number of vCPUs (default: 1)', + ) + source.add_argument( + '--parallel-readers', type=int, default=1, metavar='N', + help='number of concurrent WARC readers per process (default: 1)', + ) + + output = parser.add_argument_group('output') + output.add_argument( + '--prefix', default='REPACKAGE', + help='prefix (path) of the WARC files, local, s3:// or hf://, e.g. out/CC-HOMEPAGES (default: REPACKAGE)' + ) + output.add_argument('--subprefix', help='subprefix for the WARC filenames') + output.add_argument( + '--size', type=parse_size, default=1000000000, + help='target size of each WARC file, e.g. 1000000000, 100MB or 1GiB (default: 1GB)', + ) + output.add_argument('--creator', help='creator of the WARC: person, organization, service') + output.add_argument('--operator', help='a person, if the creator is an organization') + output.add_argument('--is-part-of', help='isPartOf of the warcinfo record (default: prefix-subprefix)') + output.add_argument('--description', help='description of the warcinfo record (default: the command line)') + output.add_argument( + '--add-metadata-record', action='append', default=[], metavar='PATH', + help='add the content of this file (local or remote) as a metadata record to each WARC (repeatable)', + ) + + advanced = parser.add_argument_group('advanced') + advanced.add_argument( + '--no-sort', action='store_true', + help=('keep the input order instead of sorting by WARC file and offset (sorting improves read ' + 'locality but keeps all captures in memory)'), + ) + advanced.add_argument( + '--log-every-n', type=int, default=1000, metavar='N', + help='log progress every N records (0 = off, default: 1000)', + ) + + parser.set_defaults(func=run) + return parser + + +def build_warcinfo(args: Namespace, cmdline: str) -> dict: + ispartof = args.is_part_of + if not ispartof: + ispartof = args.prefix + if args.subprefix: + ispartof += '-' + args.subprefix + + info = { + 'software': 'pypi_cdx_toolkit/' + get_version(), + 'isPartOf': ispartof, + 'description': args.description or 'warc extraction generated with: ' + cmdline, + 'format': 'WARC file version 1.0', # todo: if we directly read a warc, have this match the warc + } + if args.creator: + info['creator'] = args.creator + if args.operator: + info['operator'] = args.operator + return info + + +def run(args: Namespace, cmdline: str): + index_format = None if args.index_format == 'auto' else canonical_format(args.index_format) + try: + check_glob(args.input) + except ValueError as e: + raise UsageError(str(e)) from None + if index_format == 'parquet' or (index_format is None and format_from_path(args.input) == 'parquet'): + try: + import_pyarrow() + except RuntimeError as e: + raise UsageError(str(e)) from None + records = read_records(args.input, index_format) + info = build_warcinfo(args, cmdline) + start_time = time.time() + + if args.wb: + n = repackage_from_wayback(args, records, info) + else: + n = repackage_by_warc_location(args, records, info) + + LOGGER.info('wrote %d WARC records in %.1f seconds', n, time.time() - start_time) + + +def repackage_by_warc_location(args: Namespace, records, info: dict) -> int: + """Copy the WARC records byte-for-byte with the async WARC fetcher (optionally multi-process).""" + import fsspec + + from cdx_toolkit.repackage.hf_utils import is_hf_url + from cdx_toolkit.repackage.sources.records import MissingWarcLocation, RecordsSource + + try: + source = RecordsSource(records, args.warc_download_prefix, sort=not args.no_sort) + + # make sure the output directory exists + prefix_fs, prefix_fs_path = fsspec.url_to_fs(args.prefix) + prefix_fs.makedirs(prefix_fs._parent(prefix_fs_path), exist_ok=True) + + # The WARC writer supports local and S3 output; hf:// output is staged locally by the multi-process mode. + if args.processes > 1 or is_hf_url(args.prefix): + from cdx_toolkit.repackage.multiprocess import run_multiprocess_repackage + + return run_multiprocess_repackage( + source=source, + n_processes=max(1, args.processes), + readers_per_process=args.parallel_readers, + prefix_path=args.prefix, + writer_info=info, + writer_subprefix=args.subprefix, + write_paths_as_metadata_records=args.add_metadata_record, + log_every_n=args.log_every_n, + aws_region_name='us-east-1', + max_attempts=5, + uvloop=os.environ.get('CDXT_UVLOOP') == '1', + max_file_size=args.size, + warc_download_prefix=args.warc_download_prefix, + ) + + from cdx_toolkit.repackage.warc_filter import WARCFilter + + return WARCFilter( + source=source, + prefix_path=args.prefix, + writer_info=info, + writer_subprefix=args.subprefix, + write_paths_as_metadata_records=args.add_metadata_record, + log_every_n=args.log_every_n, + warc_download_prefix=args.warc_download_prefix, + n_parallel_readers=args.parallel_readers, + max_file_size=args.size, + ).filter() + except MissingWarcLocation as e: + raise UsageError(str(e)) + + +def repackage_from_wayback(args: Namespace, records, info: dict) -> int: + """Replay the captures from a wayback machine, one by one, and write them with the WARC writer of cdx_toolkit.""" + for option, used in ( + ('--processes', args.processes > 1), + ('--parallel-readers', args.parallel_readers > 1), + ('--add-metadata-record', args.add_metadata_record), + ): + if used: + raise UsageError(f'{option} cannot be used with --wb') + + wb = WAYBACK_NAMES.get(args.wb, args.wb) + n = 0 + with cdx_toolkit.warc.get_writer(args.prefix, args.subprefix, info, size=args.size) as writer: + for capture in records: + try: + record = fetch_record(capture, wb) + except RuntimeError: # pragma: no cover + LOGGER.warning('skipping capture for RuntimeError 404: %s %s', capture['url'], capture['timestamp']) + continue + writer.write_record(record) + n += 1 + return n + + +def fetch_record(capture: Mapping, wb: Optional[str]): + if capture.get('mime') == 'warc/revisit': + LOGGER.warning('revisit record being resolved for url %s %s', capture['url'], capture['timestamp']) + return fetch_wb_warc(capture, wb=wb) diff --git a/cdx_toolkit/indexes/__init__.py b/cdx_toolkit/indexes/__init__.py new file mode 100644 index 0000000..57c61ed --- /dev/null +++ b/cdx_toolkit/indexes/__init__.py @@ -0,0 +1,3 @@ +from .base import IndexBackend, UsageError, iter_index_records # noqa: F401 +from .query import IndexQuery # noqa: F401 +from .resolve import resolve_index # noqa: F401 diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py new file mode 100644 index 0000000..4d65808 --- /dev/null +++ b/cdx_toolkit/indexes/base.py @@ -0,0 +1,57 @@ +from abc import ABC, abstractmethod +import logging +from typing import FrozenSet, Iterable, Iterator, Mapping + +from .query import IndexQuery, field_flag + + +LOGGER = logging.getLogger(__name__) + + +class UsageError(ValueError): + """Invalid combination of command line options, reported without a traceback.""" + + +class IndexBackend(ABC): + """An index that can be queried for captures (CDX API, and later CDX files, Athena, DuckDB).""" + + name: str = '' + supported_options: FrozenSet[str] = frozenset() + + def validate(self, query: IndexQuery) -> None: + unsupported = sorted(query.options_used() - self.supported_options) + if unsupported: + options = ', '.join(field_flag(option) for option in unsupported) + raise UsageError(f'{options} cannot be used with the {self.name} index') + + @abstractmethod + def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: + """Yield the selected captures as mappings with CDX field names (url, timestamp, filename, ...). + + Backends apply --limit themselves, so it can be pushed down to the index.""" + raise NotImplementedError + + def count(self, query: IndexQuery) -> int: + """Cheap (possibly estimated) number of captures matching the query.""" + raise UsageError(f'--count is not supported for the {self.name} index') + + def explain(self, query: IndexQuery) -> str: + """Describe what would be queried (--dry-run), without querying the index.""" + raise UsageError(f'--dry-run is not supported for the {self.name} index') + + +def iter_index_records(backend: IndexBackend, query: IndexQuery) -> Iterator[Mapping]: + """Query a backend and apply the client-side filters that work for every backend.""" + return filter_by_url(backend.iter_records(query), query.url_fgrep, query.url_fgrepv) + + +def filter_by_url(records: Iterable[Mapping], url_fgrep=None, url_fgrepv=None) -> Iterator[Mapping]: + for record in records: + url = record['url'] + if url_fgrep and url_fgrep not in url: + LOGGER.debug('skipping due to --url-fgrep: %s', url) + continue + if url_fgrepv and url_fgrepv in url: + LOGGER.debug('skipping due to --url-fgrepv: %s', url) + continue + yield record diff --git a/cdx_toolkit/indexes/cdx_api.py b/cdx_toolkit/indexes/cdx_api.py new file mode 100644 index 0000000..6de7ff3 --- /dev/null +++ b/cdx_toolkit/indexes/cdx_api.py @@ -0,0 +1,90 @@ +import logging +from typing import Iterator, Mapping, Optional + +import cdx_toolkit +from cdx_toolkit.commoncrawl import normalize_crawl + +from .base import IndexBackend, UsageError +from .query import IndexQuery + + +LOGGER = logging.getLogger(__name__) + + +class CdxApiBackend(IndexBackend): + """A CDX API server: Common Crawl ('cc'), the Internet Archive ('ia'), or any pywb/wayback URL.""" + + name = 'CDX API' + supported_options = frozenset({ + 'urls', 'hostnames', 'domains', 'crawl', 'from_ts', 'to', 'closest', 'filters', + 'url_fgrep', 'url_fgrepv', 'limit', + }) + + def __init__(self, source: str, cc_mirror: Optional[str] = None, cc_sort: Optional[str] = None): + self.source = source + self.cc_mirror = cc_mirror + self.cc_sort = cc_sort + + def validate(self, query: IndexQuery) -> None: + super().validate(query) + if not query.url_patterns(): + raise UsageError('the CDX API needs at least one --url, --hostname or --domain') + if query.crawl and self.source != 'cc': + raise UsageError('--crawl can only be used with the Common Crawl index (cc)') + + def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: + cdx = self._fetcher(query) + remaining = query.limit + for url, match_type in query.url_patterns(): + params = self._params(query, match_type, limit=remaining) + # closest only makes sense for a single (non-paged) query + captures = cdx.get(url, **params) if query.closest else cdx.iter(url, **params) + for capture in captures: + yield capture + if remaining is not None: + remaining -= 1 + if remaining <= 0: + return + + def count(self, query: IndexQuery) -> int: + cdx = self._fetcher(query) + return sum( + cdx.get_size_estimate(url, **self._params(query, match_type, limit=query.limit)) + for url, match_type in query.url_patterns() + ) + + def explain(self, query: IndexQuery) -> str: + lines = [] + for url, match_type in query.url_patterns(): + params = ' '.join(f'{k}={v}' for k, v in self._params(query, match_type, limit=query.limit).items()) + lines.append(f'{self.source}: url={url} {params}'.rstrip()) + if query.crawl: + lines.append(f'crawls: {query.crawl}') + return '\n'.join(lines) + '\n' + + def _fetcher(self, query: IndexQuery) -> cdx_toolkit.CDXFetcher: + kwargs = {'source': self.source} + if self.cc_mirror: + kwargs['cc_mirror'] = self.cc_mirror + if self.cc_sort: + kwargs['cc_sort'] = self.cc_sort + if query.crawl: + kwargs['crawl'] = normalize_crawl([query.crawl]) + return cdx_toolkit.CDXFetcher(**kwargs) + + @staticmethod + def _params(query: IndexQuery, match_type: Optional[str], limit: Optional[int]) -> dict: + params = {} + if limit: + params['limit'] = limit + if query.from_ts: + params['from_ts'] = query.from_ts + if query.to: + params['to'] = query.to + if query.closest: + params['closest'] = query.closest + if query.filters: + params['filter'] = list(query.filters) + if match_type: + params['matchType'] = match_type + return params diff --git a/cdx_toolkit/indexes/cdx_files.py b/cdx_toolkit/indexes/cdx_files.py new file mode 100644 index 0000000..10d5eb7 --- /dev/null +++ b/cdx_toolkit/indexes/cdx_files.py @@ -0,0 +1,254 @@ +''' +Index backend for CDX files (e.g. Common Crawl's cc-index/collections/*/indexes/cdx-*.gz), local or remote. + +Each line is ` ` (CDXJ). Captures are selected by matching the SURT key +against SURT prefixes (with url-is-in) or exact SURT keys, built from --url, --hostname, --domain +and --url-file. +''' +import logging +import os +from concurrent.futures import ProcessPoolExecutor +from dataclasses import dataclass +from typing import Iterator, List, Mapping, Optional, Tuple + +from cdx_toolkit.record_io import check_glob, format_from_path, is_glob, open_text, parse_cdxj_line, read_records +from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up + +from .base import IndexBackend, UsageError +from .patterns import ( # noqa: F401 (url_pattern_to_surts, to_surt) + domain_entry_to_surts, domain_to_surts, hostname_to_surts, to_surt, url_pattern_to_surts, +) +from .query import IndexQuery + + +LOGGER = logging.getLogger(__name__) + +GLOB_CHARS = ('*', '?', '[') + +# file formats of --url-file that have columns (others: one URL pattern per line) +TABLE_FORMATS = ('csv', 'tsv', 'jsonl', 'parquet') + + +URL_FILE_TYPES = ('pattern', 'domain') + + +def load_url_patterns(path: str, column: Optional[str] = None) -> List[str]: + """URL patterns from a file: one per line (e.g. .txt), or a column of a CSV, TSV, JSONL or Parquet file.""" + fmt = format_from_path(path) + if fmt in TABLE_FORMATS: + column = column or 'url' + patterns = [] + for i, record in enumerate(read_records(path, fmt)): + if column not in record: + if i == 0: + raise UsageError(f'--url-file {path} has no column {column!r} (use --url-column)') + continue + if record[column]: + patterns.append(str(record[column])) + return patterns + if column: + raise UsageError(f'--url-column needs a CSV, TSV, JSONL or Parquet --url-file, not {path}') + with open_text(path, 'r') as fh: + return [line.strip() for line in fh if line.strip() and not line.startswith('#')] + + +@dataclass(frozen=True) +class CaptureFilter: + """Which lines of a CDX file are selected (picklable, for worker processes).""" + + prefixes: Tuple[str, ...] = () + exact: Tuple[str, ...] = () + select_all: bool = False + from_ts: Optional[str] = None + to: Optional[str] = None + url_fgrep: Optional[str] = None + url_fgrepv: Optional[str] = None + + @classmethod + def from_query(cls, query: IndexQuery) -> 'CaptureFilter': + prefixes, exact = [], [] + patterns = list(query.urls) + if query.url_file: + entries = load_url_patterns(query.url_file, query.url_column) + LOGGER.info('loaded %d entries from %s', len(entries), query.url_file) + if (query.url_file_type or 'pattern') == 'domain': + for entry in entries: + prefixes += domain_entry_to_surts(entry) + else: + patterns += entries + for pattern in patterns: + p, e = url_pattern_to_surts(pattern) + prefixes += p + exact += e + for hostname in query.hostnames: + prefixes += hostname_to_surts(hostname) + for domain in query.domains: + prefixes += domain_to_surts(domain) + return cls( + prefixes=tuple(prefixes), + exact=tuple(exact), + select_all=not (query.urls or query.url_file or query.hostnames or query.domains), + from_ts=pad_timestamp(query.from_ts) if query.from_ts else None, + to=pad_timestamp_up(query.to) if query.to else None, + url_fgrep=query.url_fgrep, + url_fgrepv=query.url_fgrepv, + ) + + def matcher(self) -> 'LineMatcher': + return LineMatcher(self) + + +class LineMatcher: + def __init__(self, capture_filter: CaptureFilter): + from url_is_in import SURTMatcher + + self.filter = capture_filter + self.exact = set(capture_filter.exact) + self.prefixes = None + if capture_filter.prefixes: + self.prefixes = SURTMatcher(list(capture_filter.prefixes), match_subdomains=False) + + def matches(self, line: str) -> bool: + key, _, rest = line.partition(' ') + f = self.filter + if not f.select_all and key not in self.exact and not (self.prefixes and self.prefixes.is_in(key)): + return False + if f.from_ts or f.to: + timestamp = rest[:rest.find(' ')] + if (f.from_ts and timestamp < f.from_ts) or (f.to and timestamp > f.to): + return False + if f.url_fgrep or f.url_fgrepv: + url = parse_cdxj_line(line).get('url', '') + if (f.url_fgrep and f.url_fgrep not in url) or (f.url_fgrepv and f.url_fgrepv in url): + return False + return True + + +def iter_matching_lines(path: str, matcher: LineMatcher) -> Iterator[str]: + n_lines = n_matches = 0 + with open_text(path, 'r') as fh: + for line in fh: + n_lines += 1 + if line.strip() and matcher.matches(line): + n_matches += 1 + yield line + LOGGER.info('%s: %d of %d lines selected', path, n_matches, n_lines) + + +def _filter_file(task: Tuple[str, str, CaptureFilter]) -> Tuple[str, int]: + """Write the selected lines of one CDX file to its own output (worker process); no file without matches.""" + input_path, output_path, capture_filter = task + matcher = capture_filter.matcher() + n = 0 + fh = ctx = None + try: + for line in iter_matching_lines(input_path, matcher): + if fh is None: + if '://' not in output_path: + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + ctx = open_text(output_path, 'w') + fh = ctx.__enter__() + fh.write(line if line.endswith('\n') else line + '\n') + n += 1 + finally: + if ctx is not None: + ctx.__exit__(None, None, None) + return output_path, n + + +class CdxFilesBackend(IndexBackend): + """CDX files (CDXJ, optionally gzipped), local or remote (fsspec), a single file or a glob.""" + + name = 'CDX files' + supported_options = frozenset({ + 'urls', 'hostnames', 'domains', 'url_file', 'url_column', 'url_file_type', 'from_ts', 'to', + 'url_fgrep', 'url_fgrepv', 'limit', + }) + + def __init__(self, location: str): + self.location = location + + def validate(self, query: IndexQuery) -> None: + super().validate(query) + try: + check_glob(self.location) + except ValueError as e: + raise UsageError(str(e)) from None + if (query.url_column or query.url_file_type) and not query.url_file: + raise UsageError('--url-column and --url-file-type need --url-file') + if query.url_file_type and query.url_file_type not in URL_FILE_TYPES: + choices = ', '.join(URL_FILE_TYPES) + raise UsageError(f'unknown --url-file-type {query.url_file_type!r} (choose from {choices})') + if not (query.urls or query.url_file or query.hostnames or query.domains or query.from_ts or query.to): + raise UsageError( + f'select captures from the {self.name} with --url, --hostname, --domain, --url-file, --from or --to' + ) + for timestamp in (query.from_ts, query.to): + if timestamp and not (timestamp.isdigit() and 4 <= len(timestamp) <= 14): + raise UsageError(f'invalid timestamp {timestamp!r}, use e.g. 2008, 20080328 or 200906302359') + for pattern in query.urls: + url_pattern_to_surts(pattern) + + def files(self) -> List[Tuple[str, str]]: + """(path, path relative to the base directory of the glob) of each CDX file.""" + import fsspec + + if not is_glob(self.location): + name = self.location.split('?', 1)[0] if '://' in self.location else self.location + return [(self.location, name.rstrip('/').rsplit('/', 1)[-1])] + fs, fs_path = fsspec.url_to_fs(self.location) + first_glob = min(fs_path.index(c) for c in GLOB_CHARS if c in fs_path) + base = fs_path[:fs_path.rfind('/', 0, first_glob)] + paths = sorted(fs.glob(fs_path)) + if not paths: + raise FileNotFoundError(f'no files found for {self.location}') + return [(fs.unstrip_protocol(p), p[len(base):].lstrip('/')) for p in paths] + + def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: + matcher = CaptureFilter.from_query(query).matcher() + remaining = query.limit + for path, _ in self.files(): + for line in iter_matching_lines(path, matcher): + yield parse_cdxj_line(line) + if remaining is not None: + remaining -= 1 + if remaining <= 0: + return + + def count(self, query: IndexQuery) -> int: + """Exact number of selected captures (reads all files).""" + matcher = CaptureFilter.from_query(query).matcher() + return sum(1 for path, _ in self.files() for _ in iter_matching_lines(path, matcher)) + + def explain(self, query: IndexQuery) -> str: + capture_filter = CaptureFilter.from_query(query) + files = self.files() + lines = [f'{len(files)} CDX file(s): {self.location}'] + lines += [f' {path}' for path, _ in files[:20]] + ([' ...'] if len(files) > 20 else []) + lines.append(f'{len(capture_filter.prefixes)} SURT prefix(es), {len(capture_filter.exact)} exact URL(s)') + lines += [f' prefix {p}' for p in capture_filter.prefixes[:20]] + lines += [f' exact {e}' for e in capture_filter.exact[:20]] + if capture_filter.from_ts or capture_filter.to: + lines.append(f'timestamps: {capture_filter.from_ts or "-"} to {capture_filter.to or "-"}') + return '\n'.join(lines) + '\n' + + def write_per_file(self, query: IndexQuery, output_base: str, processes: int = 1) -> Tuple[int, int]: + """Write the selected lines of each CDX file, unchanged, to /. + + Returns the number of output files and captures; no file is written without matches.""" + if query.limit: + raise UsageError('--limit cannot be used when writing one output per CDX file') + capture_filter = CaptureFilter.from_query(query) + join = (lambda rel: output_base.rstrip('/') + '/' + rel) if '://' in output_base else \ + (lambda rel: os.path.join(output_base, *rel.split('/'))) + tasks = [(path, join(rel), capture_filter) for path, rel in self.files()] + + if processes > 1: + with ProcessPoolExecutor(max_workers=processes) as ex: + results = list(ex.map(_filter_file, tasks)) + else: + results = [_filter_file(task) for task in tasks] + + written = [(path, n) for path, n in results if n] + LOGGER.info('wrote %d captures to %d of %d files', sum(n for _, n in written), len(written), len(tasks)) + return len(written), sum(n for _, n in written) diff --git a/cdx_toolkit/indexes/patterns.py b/cdx_toolkit/indexes/patterns.py new file mode 100644 index 0000000..dc467d5 --- /dev/null +++ b/cdx_toolkit/indexes/patterns.py @@ -0,0 +1,58 @@ +''' +URL patterns (as for the CDX API) -> SURT keys and prefixes, used to select captures from CDX files +and from the columnar index (url_surtkey). +''' +from typing import List, Tuple + +from .base import UsageError + + +def to_surt(url: str) -> str: + from url_is_in import convert_url_to_surt + + return convert_url_to_surt(url) + + +def url_pattern_to_surts(pattern: str) -> Tuple[List[str], List[str]]: + """A CDX API URL pattern -> (SURT prefixes, exact SURT keys). + + - `*.example.com`: the domain and its subdomains + - `example.com/*`, `example.com/path*`: all URLs starting with the pattern + - `example.com/path`: exactly this URL + """ + pattern = pattern.strip() + if pattern.startswith('*.'): + domain = pattern[2:] + if not domain or any(c in domain for c in '*/?'): + raise UsageError(f'invalid URL pattern {pattern!r}, a domain pattern is e.g. *.example.com') + base = to_surt(domain)[:-2] # 'com,example)/' -> 'com,example' + return [base + ')', base + ','], [] + if pattern.endswith('*'): + prefix = pattern[:-1] + if '*' in prefix: + raise UsageError(f'invalid URL pattern {pattern!r}, * can only be at the start or the end') + return [to_surt(prefix)], [] + if '*' in pattern: + raise UsageError(f'invalid URL pattern {pattern!r}, * can only be at the start or the end') + return [], [to_surt(pattern)] + + +def hostname_to_surts(hostname: str) -> List[str]: + """All URLs of exactly this host.""" + return [to_surt(hostname.strip() + '/')] + + +def domain_to_surts(domain: str) -> List[str]: + """All URLs of the domain and its subdomains.""" + return url_pattern_to_surts('*.' + domain.strip())[0] + + +def domain_entry_to_surts(entry: str) -> List[str]: + """A --url-file-type domain entry: a domain with its subdomains, or a URL prefix if it has a path.""" + entry = entry.strip() + host_and_path = entry.split('://', 1)[-1] + if '*' in entry: + raise UsageError(f'invalid domain entry {entry!r} (no wildcards with --url-file-type domain)') + if '/' not in host_and_path.rstrip('/') and '?' not in host_and_path: + return domain_to_surts(host_and_path.rstrip('/')) + return [to_surt(entry)] diff --git a/cdx_toolkit/indexes/query.py b/cdx_toolkit/indexes/query.py new file mode 100644 index 0000000..012e8f7 --- /dev/null +++ b/cdx_toolkit/indexes/query.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass, field, fields +from typing import List, Optional, Set, Tuple + + +# command line flags of the query fields, if they differ from -- +FIELD_FLAGS = { + 'urls': '--url', + 'hostnames': '--hostname', + 'domains': '--domain', + 'from_ts': '--from', + 'filters': '--filter', + 'languages': '--language', +} + + +def field_flag(name: str) -> str: + return FIELD_FLAGS.get(name, '--' + name.replace('_', '-')) + + +@dataclass +class IndexQuery: + """What to select from an index, independent of the index backend. + + URL selectors (urls, hostnames, domains, url_file) are OR-ed; all other options are AND-ed. + `sql` is a complete query for SQL backends, used instead of the other selectors. + """ + + urls: List[str] = field(default_factory=list) + hostnames: List[str] = field(default_factory=list) + domains: List[str] = field(default_factory=list) + url_file: Optional[str] = None + url_column: Optional[str] = None + url_file_type: Optional[str] = None # 'pattern' (default) or 'domain' + crawl: Optional[str] = None + from_ts: Optional[str] = None + to: Optional[str] = None + closest: Optional[str] = None + filters: List[str] = field(default_factory=list) + url_fgrep: Optional[str] = None + url_fgrepv: Optional[str] = None + limit: Optional[int] = None + languages: List[str] = field(default_factory=list) + sql: Optional[str] = None + + def url_patterns(self) -> List[Tuple[str, Optional[str]]]: + """All URL selectors as (pattern, CDX API matchType) pairs.""" + return ( + [(url, None) for url in self.urls] + + [(hostname, 'host') for hostname in self.hostnames] + + [(domain, 'domain') for domain in self.domains] + ) + + def options_used(self) -> Set[str]: + """Names of the options that are set, used to check what a backend supports.""" + return {f.name for f in fields(self) if getattr(self, f.name) not in (None, [])} diff --git a/cdx_toolkit/indexes/resolve.py b/cdx_toolkit/indexes/resolve.py new file mode 100644 index 0000000..e4c6a45 --- /dev/null +++ b/cdx_toolkit/indexes/resolve.py @@ -0,0 +1,81 @@ +from typing import Optional +from urllib.parse import urlparse + +from .base import IndexBackend, UsageError +from .cdx_api import CdxApiBackend + + +# index names that map to CDX API sources known by CDXFetcher +INDEX_NAMES = ('cc', 'ia') + +# index names that map to locations of Common Crawl's columnar URL index (queried with DuckDB) +INDEX_ALIASES = { + 'cc-url-index-s3': 's3://commoncrawl/cc-index/table/cc-main/warc/', + 'cc-url-index-hf': 'hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/', +} + +INDEX_FORMATS = ('auto', 'cdx', 'athena', 'duckdb') + +GLOB_CHARS = ('*', '?', '[') + + +def is_cdx_api_location(location: str) -> bool: + """An index name, or an http(s) URL whose last path segment has no file extension.""" + if location in INDEX_NAMES: + return True + parsed = urlparse(location) + if parsed.scheme not in ('http', 'https'): + return False + if any(c in parsed.path for c in GLOB_CHARS): + return False + last_segment = parsed.path.rstrip('/').rsplit('/', 1)[-1] + return '.' not in last_segment + + +def detect_index_format(location: str) -> str: + """Index format from the location only (no network access).""" + if location in INDEX_ALIASES or location.endswith('.parquet'): + return 'duckdb' + if location.startswith('athena://'): + return 'athena' + return 'cdx' + + +def resolve_index( + location: str, + index_format: str = 'auto', + cc_mirror: Optional[str] = None, + cc_sort: Optional[str] = None, + assume_yes: bool = False, +) -> IndexBackend: + """Map an index name or location to the backend that can query it. + + - 'cc', 'ia', or http(s) URLs without a file extension -> CDX API + - 'athena://...' -> Athena + - 'cc-url-index-s3', 'cc-url-index-hf', '*.parquet', or any path with --index-format duckdb -> DuckDB + - anything else (local or remote CDX files, globs) -> CDX files + """ + if index_format not in INDEX_FORMATS: + raise UsageError(f'unknown index format {index_format!r} (choose from {", ".join(INDEX_FORMATS)})') + if index_format == 'auto': + index_format = detect_index_format(location) + location = INDEX_ALIASES.get(location, location) + + if index_format == 'athena': + from .sql import AthenaBackend + + return AthenaBackend(location, assume_yes=assume_yes) + + if index_format == 'duckdb': + if location in INDEX_NAMES or is_cdx_api_location(location) or location.startswith('athena://'): + raise UsageError(f'{location} cannot be queried with DuckDB; use a path to the parquet index') + from .sql import DuckDbBackend + + return DuckDbBackend(location, assume_yes=assume_yes) + + if is_cdx_api_location(location): + return CdxApiBackend(location, cc_mirror=cc_mirror, cc_sort=cc_sort) + + from .cdx_files import CdxFilesBackend + + return CdxFilesBackend(location) diff --git a/cdx_toolkit/indexes/sql.py b/cdx_toolkit/indexes/sql.py new file mode 100644 index 0000000..87ce048 --- /dev/null +++ b/cdx_toolkit/indexes/sql.py @@ -0,0 +1,313 @@ +''' +Index backends for the columnar URL index (parquet), queried with SQL via AWS Athena or DuckDB. +''' +import datetime +import logging +import re +import sys +from typing import Iterator, List, Mapping, Optional, Tuple +from urllib.parse import parse_qs, urlparse + +from cdx_toolkit.repackage.sources.base import CostEstimate +from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up +from cdx_toolkit.repackage.sources.sql_base import build_sql, resolve_crawl_names + +from .base import IndexBackend, UsageError +from .patterns import url_pattern_to_surts +from .query import IndexQuery, field_flag + + +LOGGER = logging.getLogger(__name__) + +# columns of the columnar index selected by the guided (--hostname/--domain) queries, +# mapped to the field names of the CDX API so records look the same for every backend +COLUMNS = { + 'url_surtkey': 'urlkey', + 'fetch_time': 'timestamp', + 'url': 'url', + 'content_mime_type': 'mime', + 'content_mime_detected': 'mime-detected', + 'fetch_status': 'status', + 'content_digest': 'digest', + 'warc_record_length': 'length', + 'warc_record_offset': 'offset', + 'warc_filename': 'filename', + 'content_languages': 'languages', + 'content_charset': 'encoding', + 'fetch_redirect': 'redirect', + 'crawl': 'crawl', +} + +# A query restricted to at most this many crawls is considered cheap enough to +# run without a cost-confirmation prompt. +LARGE_CRAWL_SET_THRESHOLD = 10 + +GUIDED_OPTIONS = ('urls', 'hostnames', 'domains', 'crawl', 'from_ts', 'to', 'languages', 'limit') + + +def confirm_cost(estimate: Optional[CostEstimate], confirmed: bool) -> None: + """Prompt before running a potentially expensive index scan. + + A scan bounded to <= LARGE_CRAWL_SET_THRESHOLD crawls runs without a prompt. + Otherwise (no crawl filter, a large crawl set, or unverifiable raw SQL) we confirm + interactively, abort in non-interactive sessions, unless `confirmed` (--yes).""" + if confirmed or estimate is None: + return + + n_crawls = estimate.n_crawls + if n_crawls is not None and n_crawls <= LARGE_CRAWL_SET_THRESHOLD: + return + + engine = estimate.engine + if n_crawls is None: + reason = (f'This {engine} query is not restricted to specific crawls (or uses custom SQL ' + 'whose crawl partition pruning could not be verified) and may scan ALL crawls.') + else: + reason = (f'This {engine} query is restricted to {n_crawls} crawls (more than ' + f'{LARGE_CRAWL_SET_THRESHOLD}) and may scan a large amount of data.') + + if not sys.stdin.isatty(): + raise SystemExit( + reason + ' Refusing to run a potentially expensive scan in non-interactive mode. ' + f'Restrict with --crawl (<={LARGE_CRAWL_SET_THRESHOLD} crawls), or pass --yes.' + ) + + LOGGER.warning(reason) + # the prompt goes to stderr, stdout may be the output of the captures + sys.stderr.write('Index SQL scans can be expensive (Athena bills per TB scanned). Proceed? [y/N] ') + sys.stderr.flush() + answer = input() + if answer.strip().lower() not in ('y', 'yes'): + raise SystemExit('Aborted by user.') + + +def normalize_row(row: Mapping) -> dict: + """Columnar index row -> record with CDX field names; timestamps as YYYYMMDDhhmmss strings.""" + record = {} + for column, value in row.items(): + if value is None: + continue + if isinstance(value, datetime.datetime): + if value.tzinfo is not None: + value = value.astimezone(datetime.timezone.utc) + value = value.strftime('%Y%m%d%H%M%S') + elif column == 'fetch_time': + value = re.sub(r'\D', '', str(value))[:14] # Athena: '2024-07-13 10:22:33.000' + elif isinstance(value, (int, float)) and not isinstance(value, bool): + value = str(value) # the CDX API returns strings + record[COLUMNS.get(column, column)] = value + return record + + +def sql_timestamp(timestamp: str, up: bool = False) -> str: + """CDX timestamp (e.g. 2008, 200906302359) -> SQL TIMESTAMP literal of its first (or last) second.""" + if not re.fullmatch(r'\d{4,14}', timestamp): + raise UsageError(f'invalid timestamp {timestamp!r}, use e.g. 2008, 20080328 or 200906302359') + padded = pad_timestamp_up(timestamp) if up else pad_timestamp(timestamp) + try: + moment = datetime.datetime.strptime(padded, '%Y%m%d%H%M%S') + except ValueError: + raise UsageError(f'invalid timestamp {timestamp!r}') from None + return f"TIMESTAMP '{moment:%Y-%m-%d %H:%M:%S}'" + + +def check_language(code: str) -> str: + """An ISO 639-3 language code (three letters, e.g. eng or deu), in lowercase.""" + language = code.strip().lower() + if not re.fullmatch(r'[a-z]{3}', language): + raise UsageError(f'invalid language {code!r}, use ISO 639-3 codes (three letters), e.g. eng or deu') + return language + + +def url_surtkeys(query: IndexQuery) -> Tuple[List[str], List[str]]: + """--url patterns -> (url_surtkey prefixes, exact url_surtkey values).""" + prefixes, exact = [], [] + for pattern in query.urls: + p, e = url_pattern_to_surts(pattern) + prefixes += p + exact += e + return prefixes, exact + + +def query_conditions(query: IndexQuery) -> List[str]: + """WHERE predicates for --from, --to and --language.""" + conditions = [] + if query.from_ts: + conditions.append(f'fetch_time >= {sql_timestamp(query.from_ts)}') + if query.to: + conditions.append(f'fetch_time <= {sql_timestamp(query.to, up=True)}') + if query.languages: + # primary content language: the first of the comma-separated content_languages + predicates = [] + for code in query.languages: + language = check_language(code) + predicates += [f"content_languages = '{language}'", f"content_languages LIKE '{language},%'"] + conditions.append('(' + ' OR '.join(predicates) + ')') + return conditions + + +class SqlBackend(IndexBackend): + """Common logic of the SQL backends: guided query building, cost guard, row normalization.""" + + engine = '' + supported_options = frozenset({ + 'urls', 'hostnames', 'domains', 'crawl', 'from_ts', 'to', 'languages', 'limit', 'sql', 'url_fgrep', + 'url_fgrepv', + }) + + def __init__(self, location: str, assume_yes: bool = False): + self.location = location + self.assume_yes = assume_yes + self._prepared = None + + url_file_hint = '' + + def validate(self, query: IndexQuery) -> None: + if query.url_file or query.url_column or query.url_file_type: + raise UsageError( + f'--url-file cannot be used with the {self.name} index; instead, use a custom --sql query that joins ' + f'the index with a table of the target URLs, e.g.{self.url_file_hint}' + ) + super().validate(query) + if query.sql: + combined = [field_flag(option) for option in GUIDED_OPTIONS if option in query.options_used()] + if combined: + raise UsageError(f'{", ".join(combined)} cannot be combined with --sql') + elif not (query.urls or query.hostnames or query.domains): + raise UsageError(f'the {self.name} index needs --url, --hostname, --domain or --sql') + else: + url_surtkeys(query) # check --url + query_conditions(query) # check --from, --to and --language + + def from_clause(self, crawls: Optional[List[str]]) -> Tuple[str, bool]: + """FROM expression for the guided query, and whether it already prunes the crawls.""" + raise NotImplementedError + + def iter_sql_rows(self, sql: str) -> Iterator[dict]: + raise NotImplementedError + + def prepare(self, query: IndexQuery, order_by: bool = True) -> Tuple[str, Optional[int]]: + """The SQL to run and the number of crawls it is restricted to (None = unknown).""" + if query.sql: + return query.sql.strip().rstrip(';'), None + crawls = resolve_crawl_names(query.crawl) if query.crawl else None + from_clause, prunes_crawls = self.from_clause(crawls) + surtkey_prefixes, surtkeys = url_surtkeys(query) + sql = build_sql( + from_clause, + query.hostnames, + crawls=None if prunes_crawls else crawls, + limit=query.limit or 0, + url_host_registered_domains=query.domains, + order_by=order_by, + columns=list(COLUMNS), + conditions=query_conditions(query), + url_surtkey_prefixes=surtkey_prefixes, + url_surtkeys=surtkeys, + ) + return sql, len(crawls) if crawls else None + + def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: + sql, n_crawls = self.prepare(query) + confirm_cost(CostEstimate(n_crawls=n_crawls, engine=self.engine), self.assume_yes) + for row in self.iter_sql_rows(sql): + yield normalize_row(row) + + def count(self, query: IndexQuery) -> int: + sql, n_crawls = self.prepare(query, order_by=False) + confirm_cost(CostEstimate(n_crawls=n_crawls, engine=self.engine), self.assume_yes) + row = next(self.iter_sql_rows(f'SELECT count(*) AS n FROM ({sql}) AS q')) + return int(row['n']) + + def explain(self, query: IndexQuery) -> str: + sql, n_crawls = self.prepare(query) + scope = f'{n_crawls} crawl(s)' if n_crawls else 'all crawls (or not verifiable)' + return f'-- {self.name} query on {self.location}, restricted to {scope}\n{sql.strip()}\n' + + +class DuckDbBackend(SqlBackend): + """The columnar index queried with DuckDB: a hive-partitioned directory or parquet files (local, s3, hf).""" + + name = 'DuckDB' + engine = 'duckdb' + url_file_hint = ( + ':\n\n' + ' SELECT idx.url, idx.warc_filename, idx.warc_record_offset, idx.warc_record_length\n' + " FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-33/subset=warc/" + "*.parquet') AS idx\n" + " JOIN read_csv('urls.csv') AS target ON idx.url = target.url" + ) + + def __init__(self, location: str, assume_yes: bool = False): + if "'" in location: + raise UsageError(f'invalid index location {location!r}') + super().__init__(location, assume_yes) + + def from_clause(self, crawls): + from cdx_toolkit.repackage.sources.duckdb import _build_from_clause + + if self.location.endswith('.parquet'): # file(s), possibly a glob + return f"read_parquet('{self.location}', hive_partitioning=true)", False + # hive-partitioned index directory: prune the crawls in the glob + return _build_from_clause(self.location, crawls), True + + def iter_sql_rows(self, sql): + from cdx_toolkit.repackage.sources.duckdb import DuckDbSource + + source = DuckDbSource(query=sql, index_path=self.location, warc_download_prefix=None) + return source.iter_rows() + + +class AthenaBackend(SqlBackend): + """The columnar index queried with AWS Athena. + + Location: athena://athena..amazonaws.com[:443]/?S3OutputLocation=s3://...[&table=ccindex] + """ + + name = 'Athena' + engine = 'athena' + url_file_hint = ( + ' (after creating an Athena table `target_urls` from the URL file on S3):\n\n' + ' SELECT idx.url, idx.warc_filename, idx.warc_record_offset, idx.warc_record_length\n' + ' FROM ccindex AS idx\n' + ' JOIN target_urls AS target ON idx.url = target.url\n' + " WHERE idx.crawl = 'CC-MAIN-2024-33' AND idx.subset = 'warc'" + ) + + def __init__(self, location: str, assume_yes: bool = False): + super().__init__(location, assume_yes) + self.region, self.database, self.table, self.s3_output_location = parse_athena_location(location) + + def from_clause(self, crawls): + return self.table, False + + def iter_sql_rows(self, sql): + from cdx_toolkit.repackage.sources.athena import AthenaSource + + source = AthenaSource( + query=sql, + database=self.database, + s3_output_location=self.s3_output_location, + warc_download_prefix=None, + region_name=self.region, + ) + return source.iter_rows() + + +def parse_athena_location(location: str) -> Tuple[str, str, str, str]: + """athena://athena.us-east-1.amazonaws.com:443/ccindex?S3OutputLocation=s3://bucket/path/ + -> (region, database, table, S3 output location)""" + parsed = urlparse(location) + match = re.fullmatch(r'athena\.([a-z0-9-]+)\.amazonaws\.com', parsed.hostname or '') + if parsed.scheme != 'athena' or not match: + raise UsageError( + f'invalid Athena location {location!r}, expected ' + 'athena://athena..amazonaws.com/?S3OutputLocation=s3:////' + ) + params = parse_qs(parsed.query) + s3_output_location = params.get('S3OutputLocation', [None])[0] + if not s3_output_location or not s3_output_location.startswith('s3://'): + raise UsageError(f'the Athena location {location!r} needs a S3OutputLocation=s3://... parameter') + database = parsed.path.strip('/') or 'ccindex' + table = params.get('table', ['ccindex'])[0] + return match.group(1), database, table, s3_output_location diff --git a/cdx_toolkit/record_io.py b/cdx_toolkit/record_io.py new file mode 100644 index 0000000..12ca453 --- /dev/null +++ b/cdx_toolkit/record_io.py @@ -0,0 +1,281 @@ +''' +Reading and writing index records (captures) as JSONL, CDXJ, CSV, TSV or Parquet. + +Paths can be local or remote (via fsspec, if installed), optionally gzipped; "-" is stdin/stdout. +Parquet needs the optional pyarrow dependency (cdx_toolkit[parquet]) and a file path. +''' +import contextlib +import csv +import gzip +import itertools +import json +import re +import sys +from typing import IO, Iterable, Iterator, List, Mapping, Optional + +try: + import fsspec +except ImportError: # pragma: no cover - exercised in minimal installs + fsspec = None + + +TEXT_FORMATS = ('jsonl', 'cdxj', 'csv', 'tsv') +FORMATS = TEXT_FORMATS + ('parquet',) + +# Common Crawl's index files are named *.cdx(.gz) but use the CDXJ format (crawl index JSON) +FORMAT_ALIASES = {'cdx': 'cdxj'} +# formats for the command line, including the aliases +FORMAT_CHOICES = ('jsonl', 'cdx', 'cdxj', 'csv', 'tsv', 'parquet') + +# records per Parquet row group +PARQUET_BATCH_SIZE = 100_000 + +EXTENSION_FORMATS = { + '.jsonl': 'jsonl', + '.json': 'jsonl', + '.cdxj': 'cdxj', + '.cdx': 'cdxj', + '.csv': 'csv', + '.tsv': 'tsv', + '.parquet': 'parquet', +} + +# column names of the columnar index (and range-job CSVs) -> CDX field names +FIELD_ALIASES = { + 'warc_filename': 'filename', + 'warc_record_offset': 'offset', + 'warc_record_length': 'length', +} + +CDXJ_LINE = re.compile(r'^\S+ \d{4,14} \{') + + +def canonical_format(fmt: Optional[str]) -> Optional[str]: + """'cdx' -> 'cdxj'; other formats unchanged.""" + return FORMAT_ALIASES.get(fmt, fmt) + + +def format_from_path(path: str) -> Optional[str]: + """Infer the record format from a file extension (ignoring .gz), None if unknown.""" + if path == '-': + return None + path = path.lower() + if path.endswith('.gz'): + path = path[:-3] + for extension, fmt in EXTENSION_FORMATS.items(): + if path.endswith(extension): + return fmt + return None + + +def open_text(path: str, mode: str) -> IO: + """Open a local or remote text file; '-' means stdin/stdout (which is not closed).""" + if path == '-': + return contextlib.nullcontext(sys.stdout if 'w' in mode else sys.stdin) + if fsspec is not None: + return fsspec.open(path, mode + 't', compression='infer', newline='') + if '://' in path: + raise RuntimeError(f'Reading or writing {path} requires fsspec. Install cdx_toolkit[s3].') + if path.endswith('.gz'): + return gzip.open(path, mode + 't', newline='') + return open(path, mode, newline='') + + +class RecordWriter: + """Write records in one of TEXT_FORMATS; `fields` selects and orders the fields (default: all).""" + + def __init__(self, fh: IO, fmt: str, fields: Optional[List[str]] = None): + fmt = canonical_format(fmt) + if fmt not in TEXT_FORMATS: + raise ValueError(f'unknown format {fmt!r}') + self.fh = fh + self.fmt = fmt + self.fields = fields + self._csv_writer = None + + def write(self, record: Mapping) -> None: + record = dict(record) + selected = {k: record[k] for k in self.fields if k in record} if self.fields else record + + if self.fmt == 'jsonl': + self.fh.write(json.dumps(selected, sort_keys=True, default=str) + '\n') + elif self.fmt == 'cdxj': + for key in ('urlkey', 'timestamp'): + if key not in record: + raise ValueError(f'cannot write CDXJ: record has no {key!r} field') + data = {k: v for k, v in selected.items() if k not in ('urlkey', 'timestamp')} + self.fh.write(f'{record["urlkey"]} {record["timestamp"]} {json.dumps(data, default=str)}\n') + else: + if self._csv_writer is None: + self._csv_writer = csv.DictWriter( + self.fh, + fieldnames=self.fields or list(record.keys()), + delimiter='\t' if self.fmt == 'tsv' else ',', + lineterminator='\n', + extrasaction='ignore', + ) + self._csv_writer.writeheader() + self._csv_writer.writerow(selected) + + +def is_glob(path: str) -> bool: + # in http(s) URLs, ? starts the query string + chars = '*[' if path.lower().startswith(('http://', 'https://')) else '*?[' + return path != '-' and any(c in path for c in chars) + + +def check_glob(path: str) -> None: + """Globs need a filesystem that can list files: not http(s).""" + if is_glob(path) and path.lower().startswith(('http://', 'https://')): + raise ValueError( + f'globs do not work with http(s) URLs ({path}), because they cannot list files; ' + 'use e.g. s3://commoncrawl/... (needs AWS credentials) or one URL per file' + ) + + +def expand_paths(path: str) -> List[str]: + """A local or remote path, or all files matching a glob (e.g. s3://bucket/indexes/*.cdx.gz).""" + if not is_glob(path): + return [path] + check_glob(path) + if fsspec is None: + import glob + + if '://' in path: + raise RuntimeError(f'Reading {path} requires fsspec. Install cdx_toolkit[s3].') + return sorted(glob.glob(path)) + fs, fs_path = fsspec.url_to_fs(path) + return [fs.unstrip_protocol(p) for p in sorted(fs.glob(fs_path))] + + +def read_records(path: str, fmt: Optional[str] = None) -> Iterator[dict]: + """Read records from a file, all files matching a glob, or stdin ('-'). + + The format is detected from the first line of each file if not given.""" + fmt = canonical_format(fmt) + paths = expand_paths(path) + if not paths: + raise FileNotFoundError(f'no files found for {path}') + for p in paths: + if fmt == 'parquet' or (fmt is None and format_from_path(p) == 'parquet'): + yield from read_parquet(p) + else: + yield from _read_records(p, fmt) + + +def _read_records(path: str, fmt: Optional[str] = None) -> Iterator[dict]: + with open_text(path, 'r') as fh: + lines = iter(fh) + first_line = next(lines, None) + if first_line is None: + return + lines = itertools.chain([first_line], lines) + fmt = fmt or sniff_format(first_line) + + if fmt == 'jsonl': + records = (json.loads(line) for line in lines if line.strip()) + elif fmt == 'cdxj': + records = (parse_cdxj_line(line) for line in lines if line.strip()) + elif fmt in ('csv', 'tsv'): + records = csv.DictReader(lines, delimiter='\t' if fmt == 'tsv' else ',') + else: + raise ValueError(f'unknown format {fmt!r}') + + for record in records: + yield normalize_field_names(record) + + +def sniff_format(line: str) -> str: + if line.lstrip().startswith('{'): + return 'jsonl' + if CDXJ_LINE.match(line): + return 'cdxj' + if '\t' in line: + return 'tsv' + return 'csv' + + +def parse_cdxj_line(line: str) -> dict: + urlkey, timestamp, data = line.split(' ', 2) + record = json.loads(data) + record.setdefault('urlkey', urlkey) + record.setdefault('timestamp', timestamp) + return record + + +def normalize_field_names(record: dict) -> dict: + for alias, name in FIELD_ALIASES.items(): + if alias in record and name not in record: + record[name] = record.pop(alias) + return record + + +def import_pyarrow(): + try: + import pyarrow + import pyarrow.parquet + except ImportError: + raise RuntimeError( + 'Parquet files need the optional pyarrow dependency. Install cdx_toolkit[parquet].' + ) from None + return pyarrow + + +def write_parquet(records: Iterable[Mapping], path: str, fields: Optional[List[str]] = None) -> int: + """Write records to a Parquet file (all columns are strings), returns the number of records written. + + Without `fields`, the columns are the fields of the first batch of records.""" + if path == '-': + raise ValueError('Parquet output cannot be written to stdout, please set --output') + pa = import_pyarrow() + + n = 0 + schema = None + writer = None + with fsspec.open(path, 'wb') if fsspec is not None else open(path, 'wb') as fh: + try: + batch = [] + for record in itertools.chain(records, [None]): + if record is not None: + batch.append(dict(record)) + if len(batch) < PARQUET_BATCH_SIZE: + continue + if not batch and schema is not None: + break + if schema is None: + columns = fields or list(dict.fromkeys(k for r in batch for k in r)) + schema = pa.schema([(c, pa.string()) for c in columns]) + writer = pa.parquet.ParquetWriter(fh, schema) + table = pa.table( + {c: [None if r.get(c) is None else str(r[c]) for r in batch] for c in schema.names}, + schema=schema, + ) + writer.write_table(table) + n += len(batch) + batch = [] + finally: + if writer is not None: + writer.close() + return n + + +def read_parquet(path: str) -> Iterator[dict]: + pa = import_pyarrow() + with fsspec.open(path, 'rb') if fsspec is not None else open(path, 'rb') as fh: + for batch in pa.parquet.ParquetFile(fh).iter_batches(): + for record in batch.to_pylist(): + yield normalize_field_names({k: v for k, v in record.items() if v is not None}) + + +def write_records(records: Iterable[Mapping], path: str, fmt: str, fields: Optional[List[str]] = None) -> int: + """Write records to a file or stdout ('-'), returns the number of records written.""" + fmt = canonical_format(fmt) + if fmt == 'parquet': + return write_parquet(records, path, fields) + n = 0 + with open_text(path, 'w') as fh: + writer = RecordWriter(fh, fmt, fields) + for record in records: + writer.write(record) + n += 1 + return n diff --git a/cdx_toolkit/repackage/__init__.py b/cdx_toolkit/repackage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cdx_toolkit/repackage/data_classes.py b/cdx_toolkit/repackage/data_classes.py new file mode 100644 index 0000000..3465665 --- /dev/null +++ b/cdx_toolkit/repackage/data_classes.py @@ -0,0 +1,121 @@ +import time +from dataclasses import dataclass, field + +from cdx_toolkit.repackage.s3_utils import is_s3_url, parse_s3_uri, with_retries +from cdx_toolkit.repackage.hf_utils import is_hf_url +from typing import Optional, Tuple + +from cdx_toolkit.myrequests import myrequests_get + + +@dataclass +class ThroughputTracker: + """Track throughput metrics for fetchers and consumers.""" + + start_time: float = 0.0 + total_bytes: int = 0 + total_requests: int = 0 + total_records: int = 0 + + def start(self): + self.start_time = time.time() + + def add(self, bytes_count: int = 0, records_count: int = 0, requests_count: int = 1): + self.total_bytes += bytes_count + self.total_requests += requests_count + self.total_records += records_count + + def get_stats(self) -> dict: + elapsed = time.time() - self.start_time + + return { + 'elapsed': elapsed, + 'total_bytes': self.total_bytes, + 'total_requests': self.total_requests, + 'total_records': self.total_records, + 'bytes_per_sec': self.total_bytes / elapsed if elapsed > 0 else 0, + 'mb_per_sec': (self.total_bytes / elapsed) / (1024 * 1024) if elapsed > 0 else 0, + 'requests_per_sec': self.total_requests / elapsed if elapsed > 0 else 0, + 'records_per_sec': self.total_records / elapsed if elapsed > 0 else 0, + } + + +@dataclass(frozen=True) +class RangeJob: + """Defines a S3 or HTTP range read request.""" + + url: str + offset: int + length: int + records_count: int = 1 + # Relative WARC filename (e.g. crawl-data/...warc.gz) as known by the source, + # used when materializing a non-self-contained range-jobs CSV. `url` stays + # authoritative for fetching. + filename: Optional[str] = None + # Extra columns from a raw SQL --query (e.g. content_languages), kept only for + # CSV materialization / analysis. Never used for fetching. Excluded from + # equality/hash so RangeJob stays hashable despite the dict value. + extra: Optional[dict] = field(default=None, compare=False) + + def is_s3(self): + return is_s3_url(self.url) + + def get_s3_bucket_and_key(self) -> Tuple[str, str]: + if self.is_s3(): + return parse_s3_uri(self.url) + else: + raise ValueError('Cannot get bucket and key from a HTTP job') + + def is_hf(self): + return is_hf_url(self.url) + + async def ranged_get_bytes( + self, + max_attempts: int, + base_backoff_seconds: float, + s3_client=None, + hf_reader=None, + ) -> bytes: + """Ranged get request to S3 (aioboto3), an HF bucket, or HTTP.""" + offset = self.offset + length = self.length + + end = offset + length - 1 # inclusive + + if self.is_s3(): + # read from S3 + bucket, key = self.get_s3_bucket_and_key() + resp = await with_retries( + lambda: s3_client.get_object(Bucket=bucket, Key=key, Range=f'bytes={offset}-{end}'), + op_name=f'ranged_get {bucket}/{key}[{offset}:{end}]', + max_attempts=max_attempts, + base_backoff_seconds=base_backoff_seconds, + ) + return await resp['Body'].read() + + elif self.is_hf(): + # read from a Hugging Face bucket (hf://) via the configured HF reader + if hf_reader is None: + raise ValueError('hf:// range read requires an hf_reader (none was provided)') + return await hf_reader.ranged_get( + self.url, + offset, + length, + max_attempts=max_attempts, + base_backoff_seconds=base_backoff_seconds, + ) + + else: + # read from HTTP + headers = {'Range': 'bytes={}-{}'.format(offset, end)} + + resp = myrequests_get(self.url, headers=headers) + return resp.content + + +@dataclass(frozen=True) +class RangePayload: + """Bytes output from S3 or HTTP range read.""" + + job: RangeJob + data: bytes diff --git a/cdx_toolkit/repackage/hf_utils.py b/cdx_toolkit/repackage/hf_utils.py new file mode 100644 index 0000000..e197b4c --- /dev/null +++ b/cdx_toolkit/repackage/hf_utils.py @@ -0,0 +1,262 @@ +"""Hugging Face Storage Bucket support for repackage reads/writes. + +HF Storage Buckets (``hf://buckets///``) expose data only +through the Hugging Face filesystem / HTTP resolve API -- there is no S3 API yet. +So cdxt cannot reach a bucket via the aioboto3 read path. Instead WARC byte ranges +are fetched with async ``aiohttp`` ranged ``GET`` requests against the CDN: each WARC +file is resolved once (``HEAD https://huggingface.co/buckets///resolve/`` +-> redirect to a signed CDN URL, valid for about an hour), and all records of the file +are then read from the signed URL. Every ``/resolve/`` request counts against the Hub's +"Resolvers" rate limit (e.g. 5,000 per 5 minutes for free users, see +https://huggingface.co/docs/hub/en/rate-limits), the CDN requests do not. + +The records are byte-identical to the same records read from ``s3://commoncrawl``. +(Reading with ``HfFileSystem.cat_file`` instead was ~10x slower, used ~12x more memory +and dropped records under load, see the cdxt-benchmark read-backend study.) +""" +import asyncio +import collections +import contextlib +import logging +import re +import time +from typing import Tuple +from urllib.parse import parse_qs, urlparse + +logger = logging.getLogger(__name__) + +HF_SCHEME = 'hf://' +HF_ENDPOINT = 'https://huggingface.co' + + +def is_hf_url(url) -> bool: + """True for an ``hf://`` URL (e.g. an hf:// WARC download prefix or job URL).""" + return bool(url) and url.startswith(HF_SCHEME) + + +def hf_fs_path(url: str) -> str: + """``hf://buckets/ns/name/key`` -> ``buckets/ns/name/key`` (HfFileSystem path).""" + if not is_hf_url(url): + raise ValueError(f'Not an hf:// URL: {url}') + return url[len(HF_SCHEME):] + + +def hf_resolve_url(url: str) -> str: + """Map an ``hf://`` bucket URL to its CDN-fronted HTTP(S) resolve URL. + + ``hf://buckets///`` + -> ``https://huggingface.co/buckets///resolve/`` + """ + path = hf_fs_path(url) + parts = path.split('/', 3) + if len(parts) < 4 or parts[0] != 'buckets': + raise ValueError( + f'Unsupported hf:// URL (expected hf://buckets///): {url}' + ) + repo = '/'.join(parts[:3]) # buckets// + key = parts[3] + return f'{HF_ENDPOINT}/{repo}/resolve/{key}' + + +def retry_after_seconds(headers, default: float) -> float: + """Seconds to wait after a 429, from the response headers. + + Prefers ``Retry-After`` (delta-seconds); else the ``t=`` field of the + standardized ``RateLimit`` header (seconds until the window resets); else + ``default``. + """ + ra = headers.get('Retry-After') + if ra: + try: + return max(0.0, float(ra)) + except (TypeError, ValueError): + pass + m = re.search(r'(?:^|[;,\s])t\s*=\s*(\d+)', headers.get('RateLimit', '') or '') + if m: + return float(m.group(1)) + return default + + +def signed_url_expiry(url: str, default_ttl: float) -> float: + """Expiry (epoch seconds) of a signed CDN URL from its ``Expires`` parameter.""" + expires = parse_qs(urlparse(url).query).get('Expires', [None])[0] + if expires and expires.isdigit(): + return float(expires) + return time.time() + default_ttl + + +class HfHttpReader: + """Async ranged ``GET`` requests against the HF CDN (aiohttp), resolving each WARC file once. + + The ``/resolve/`` URL of a WARC file is requested once (``HEAD``, no redirect + followed) to get the signed CDN URL it redirects to; that URL is cached (LRU) and + all records of the file are fetched from it with ``Range`` requests. The URL is + resolved again shortly before it expires, or when the CDN rejects it (401/403/410). + + Only the resolve requests count against the HF Resolvers rate limit. On ``429`` the + reader honours the server-provided wait (``Retry-After`` / ``RateLimit`` header) + instead of failing, and tallies the throttle events/wait so a run reports whether it + was rate-limited. + """ + + # Bound the wait-out so a job can ride out a 5-minute window but not hang + # forever if the limit never clears. + MAX_THROTTLE_WAIT = 360.0 # cap a single 429 backoff (seconds) + MAX_THROTTLE_RETRIES = 60 # consecutive 429s before giving up on a record + + # Resolve again this many seconds before the signed URL expires. + EXPIRY_MARGIN = 300.0 + # Lifetime assumed for a signed URL without an ``Expires`` parameter. + DEFAULT_TTL = 600.0 + # Number of signed URLs kept; the input is sorted by WARC file, so few are in use at a time. + MAX_CACHED_URLS = 4096 + + def __init__(self, session, token=None): + self.session = session + self.token = token + self.throttle_count = 0 # number of 429s seen + self.throttle_wait_s = 0.0 # cumulative time spent backing off 429s + self.resolve_count = 0 # number of /resolve/ requests + self.read_count = 0 # number of records read + self._policy_logged = False + # resolve URL -> (signed CDN URL, expiry); a lock per resolve URL so concurrent + # readers of the same file share one resolve request + self._signed_urls: 'collections.OrderedDict[str, Tuple[str, float]]' = collections.OrderedDict() + self._locks = {} + + def _auth_headers(self) -> dict: + return {'Authorization': f'Bearer {self.token}'} if self.token else {} + + async def _wait_for_rate_limit(self, resp, throttles: int, base_backoff_seconds: float, url: str) -> None: + self.throttle_count += 1 + default = min(30.0, base_backoff_seconds * 2 ** min(throttles, 6)) + wait = retry_after_seconds(resp.headers, default) + if not self._policy_logged: + self._policy_logged = True + logger.warning( + 'HF resolver rate limit hit (429); RateLimit=%r Policy=%r. ' + 'Throughput is now capped by the Resolvers quota.', + resp.headers.get('RateLimit'), + resp.headers.get('RateLimit-Policy'), + ) + if throttles > self.MAX_THROTTLE_RETRIES or wait > self.MAX_THROTTLE_WAIT: + raise OSError(f'HF 429: gave up after {throttles} throttle waits (last wait {wait:.0f}s) for {url}') + self.throttle_wait_s += wait + await asyncio.sleep(wait) + + async def _resolve(self, resolve: str, *, max_attempts: int, base_backoff_seconds: float) -> Tuple[str, float]: + """Resolve a file to its signed CDN URL and the URL's expiry.""" + attempt = 0 + throttles = 0 + while True: + try: + async with self.session.head(resolve, headers=self._auth_headers(), allow_redirects=False) as resp: + if resp.status == 429: + throttles += 1 + await self._wait_for_rate_limit(resp, throttles, base_backoff_seconds, resolve) + continue # does not consume the hard-error attempt budget + self.resolve_count += 1 + location = resp.headers.get('Location') + if resp.status in (301, 302, 303, 307, 308) and location: + return location, signed_url_expiry(location, self.DEFAULT_TTL) + if resp.status == 200: + # served without a redirect: read from the resolve URL itself + return resolve, time.time() + self.DEFAULT_TTL + raise OSError(f'HF HEAD {resolve} -> {resp.status}') + except Exception: # noqa: BLE001 - retry any transient resolve error + attempt += 1 + if attempt >= max_attempts: + raise + await asyncio.sleep(min(8.0, base_backoff_seconds * 2 ** (attempt - 1))) + + async def _signed_url(self, resolve: str, *, refresh: bool, **kwargs) -> str: + lock = self._locks.setdefault(resolve, asyncio.Lock()) + async with lock: + cached = self._signed_urls.get(resolve) + if cached and not refresh and cached[1] - time.time() > self.EXPIRY_MARGIN: + self._signed_urls.move_to_end(resolve) + return cached[0] + url, expiry = await self._resolve(resolve, **kwargs) + self._signed_urls[resolve] = (url, expiry) + self._signed_urls.move_to_end(resolve) + while len(self._signed_urls) > self.MAX_CACHED_URLS: + old, _ = self._signed_urls.popitem(last=False) + self._locks.pop(old, None) + return url + + async def ranged_get( + self, + url: str, + offset: int, + length: int, + *, + max_attempts: int = 5, + base_backoff_seconds: float = 0.5, + ) -> bytes: + resolve = hf_resolve_url(url) + end = offset + length - 1 # inclusive HTTP Range + attempt = 0 + throttles = 0 + refresh = False + while True: + try: + signed_url = await self._signed_url( + resolve, refresh=refresh, max_attempts=max_attempts, base_backoff_seconds=base_backoff_seconds + ) + refresh = False + headers = {'Range': f'bytes={offset}-{end}'} + if signed_url.startswith(HF_ENDPOINT): + headers.update(self._auth_headers()) + async with self.session.get(signed_url, headers=headers) as resp: + if resp.status == 429: + throttles += 1 + await self._wait_for_rate_limit(resp, throttles, base_backoff_seconds, signed_url) + continue # does not consume the hard-error attempt budget + if resp.status in (401, 403, 410): + # signed URL expired or revoked: resolve the file again + await resp.release() + refresh = True + raise OSError(f'HF CDN GET -> {resp.status}, resolving {resolve} again') + if resp.status not in (200, 206): + body = await resp.read() + raise OSError(f'HF GET {resolve} -> {resp.status}: {body[:200]!r}') + data = await resp.read() + if len(data) != length: + raise OSError(f'HF GET {resolve} returned {len(data)} bytes, expected {length}') + self.read_count += 1 + return data + except Exception: # noqa: BLE001 - retry any transient read error + attempt += 1 + if attempt >= max_attempts: + raise + await asyncio.sleep(min(8.0, base_backoff_seconds * 2 ** (attempt - 1))) + + async def aclose(self): + logger.info('HF reader: %d records read with %d resolve requests', self.read_count, self.resolve_count) + if self.throttle_count: + logger.warning( + 'HF resolver throttling: %d x 429, %.1fs total spent backing off ' + '(read throughput was rate-limit bound)', + self.throttle_count, + self.throttle_wait_s, + ) + + +@contextlib.asynccontextmanager +async def make_hf_reader(): + """Async context manager yielding an HF reader that owns an ``aiohttp`` session. + + Imports are lazy so the optional ``hf`` extra is only required when actually + reading from a bucket. + """ + import aiohttp + from huggingface_hub import get_token + + timeout = aiohttp.ClientTimeout(total=300, sock_connect=15) + connector = aiohttp.TCPConnector(limit=0, ttl_dns_cache=300) + async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session: + reader = HfHttpReader(session, token=get_token()) + try: + yield reader + finally: + await reader.aclose() diff --git a/cdx_toolkit/repackage/local_writer.py b/cdx_toolkit/repackage/local_writer.py new file mode 100644 index 0000000..8fd9e48 --- /dev/null +++ b/cdx_toolkit/repackage/local_writer.py @@ -0,0 +1,37 @@ +import aiofiles + + +class LocalFileWriter: + """Async writer for local file system using aiofiles.""" + + def __init__(self, file_path: str, buffer_size: int = 8192, mode: str = 'wb'): + self.file_path = file_path + self.buffer_size = buffer_size + self.mode = mode + self.file_handle = None + self.buffer = bytearray() + + async def start(self): + self.file_handle = await aiofiles.open(self.file_path, self.mode) + + async def write(self, data: bytes): + self.buffer.extend(data) + if len(self.buffer) >= self.buffer_size: + await self._flush() + + async def _flush(self): + if self.buffer and self.file_handle: + await self.file_handle.write(bytes(self.buffer)) + await self.file_handle.flush() + self.buffer.clear() + + async def close(self): + try: + if self.buffer: + await self._flush() + if self.file_handle: + await self.file_handle.close() + except Exception: + if self.file_handle: + await self.file_handle.close() + raise diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py new file mode 100644 index 0000000..13474e1 --- /dev/null +++ b/cdx_toolkit/repackage/multiprocess.py @@ -0,0 +1,206 @@ +"""Multi-process repackage: shard the range jobs and fetch each shard in its own process +(one asyncio event loop per CPU core), all writing into one series of output WARCs. + +A single asyncio loop saturates one CPU core on many small range-GETs, so reaching +all cores needs multiple processes. This orchestrator keeps that entirely inside +``cdxt repackage`` (no external driver): + +1. Drain the configured source once and split it into N self-contained shard CSVs, + hashed by WARC filename so every record of a file stays in one shard. +2. Run one worker process per (non-empty) shard concurrently. Each worker **rotates its + output at ``--size``**; every output file is a self-contained WARC with its own + ``warcinfo`` record. +3. The file numbers come from **one counter shared by all workers**: whenever a worker + starts a new file it takes the next number, so the files are written under their final + names ``[-]-001.warc.gz``, ``-002`` … right away (and the ``warcinfo`` + record names the file it is in). Which worker writes which number depends on timing. + Output honours ``--size`` as single-process mode does (files are a target size; the last + file of each worker may be short, so up to N short files appear). + +The WARC writer supports local and S3 output. For other fsspec destinations (e.g. an +``hf://`` bucket) the files are staged in a local directory under their final names and +copied to the destination afterwards. +""" +import glob +import hashlib +import logging +import multiprocessing +import os +import shutil +import tempfile +import time + +import fsspec +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor +from typing import Dict, List, Optional, Tuple + +from cdx_toolkit.repackage.sources.csv import CsvSource, RangeJobCsvWriter +from cdx_toolkit.repackage.warc_filter import WARCFilter +from cdx_toolkit.repackage.warc_utils import generate_warc_filename +from cdx_toolkit.repackage.s3_utils import is_s3_url, parse_s3_uri + +logger = logging.getLogger(__name__) + +# Concurrency for copying staged files to a non-S3 remote destination (override with CDXT_COPY_CONCURRENCY). +_COPY_CONCURRENCY = int(os.environ.get('CDXT_COPY_CONCURRENCY', '16')) + +# The shared counter of output file numbers, set in each worker process by _init_worker. +_FILE_SEQUENCE = None + + +def _output_filename(prefix_path: str, sequence: int, subprefix: Optional[str] = None, + gzip: bool = True) -> str: + """Build a WARC output path for a given rotation sequence (S3 uri or local path).""" + if is_s3_url(prefix_path): + bucket, key_prefix = parse_s3_uri(prefix_path) + key = generate_warc_filename(key_prefix, sequence=sequence, writer_subprefix=subprefix, gzip=gzip) + return f's3://{bucket}/{key}' + return generate_warc_filename(prefix_path, sequence=sequence, writer_subprefix=subprefix, gzip=gzip) + + +def _init_worker(file_sequence) -> None: + global _FILE_SEQUENCE + _FILE_SEQUENCE = file_sequence + + +def _next_file_sequence(_current: int) -> int: + """Take the next output file number from the counter shared by all workers.""" + with _FILE_SEQUENCE.get_lock(): + _FILE_SEQUENCE.value += 1 + return _FILE_SEQUENCE.value + + +def _run_worker(cfg: Dict) -> int: + """Worker entrypoint (runs in a child process): fetch one shard into the output WARCs.""" + if cfg.get('uvloop'): + os.environ['CDXT_UVLOOP'] = '1' + source = CsvSource(cfg['shard_csv'], warc_download_prefix=None, sort=True) + wf = WARCFilter( + source=source, + prefix_path=cfg['prefix_path'], + writer_info=cfg['writer_info'], + writer_subprefix=cfg['subprefix'], + write_paths_as_metadata_records=cfg['write_paths_as_metadata_records'], + log_every_n=cfg['log_every_n'], + # Shard CSVs hold full URLs (warc_filename), so CsvSource ignores this; it is + # still needed so WARCFilter knows the read scheme (S3 vs hf://). + warc_download_prefix=cfg['warc_download_prefix'], + n_parallel_readers=cfg['readers'], + aws_region_name=cfg['aws_region_name'], + max_attempts=cfg['max_attempts'], + max_file_size=cfg['max_file_size'], # rotate at --size + next_file_sequence=_next_file_sequence, + ) + return wf.filter() + + +def _split_source_to_shards(source, n: int, out_dir: str) -> Tuple[List[str], List[int]]: + """Drain the source once into N self-contained shard CSVs, hashed by WARC file. + + Returns the shard paths and the number of range jobs in each shard.""" + paths = [os.path.join(out_dir, f'shard.{i}.csv') for i in range(n)] + writers = [RangeJobCsvWriter(p) for p in paths] + counts = [0] * n + total = 0 + try: + for job in source.iter_range_jobs(): + key = (job.filename or job.url or '').encode() + idx = int(hashlib.md5(key).hexdigest(), 16) % n + writers[idx].write(job) + counts[idx] += 1 + total += 1 + finally: + for w in writers: + w.close() + logger.info('Sharded %d range jobs into %d shards: %s', total, n, counts) + return paths, counts + + +def run_multiprocess_repackage( + *, + source, + n_processes: int, + readers_per_process: int, + prefix_path: str, + writer_info: Dict, + writer_subprefix: Optional[str], + write_paths_as_metadata_records: Optional[List[str]], + log_every_n: int, + aws_region_name: str, + max_attempts: int, + uvloop: bool, + max_file_size: Optional[int] = 1_000_000_000, + warc_download_prefix: Optional[str] = None, +) -> int: + """Shard -> N worker processes (rotating at --size) writing one global + [-]-NNN.warc.gz series. Returns the record count.""" + tmp_dir = tempfile.mkdtemp(prefix='cdxt_shards_') + start = time.time() + try: + shard_csvs, counts = _split_source_to_shards(source, n_processes, tmp_dir) + # empty shards would only produce WARC files without records + shard_csvs = [path for path, count in zip(shard_csvs, counts) if count > 0] or shard_csvs[:1] + + # The WARC writer supports S3 and local FS only. Other fsspec destinations + # (e.g. an hf:// bucket) are staged in a local dir and copied afterwards. + remote_non_s3 = ('://' in prefix_path) and not is_s3_url(prefix_path) + staging_dir = os.path.join(tmp_dir, 'out') + if remote_non_s3: + os.makedirs(staging_dir) + write_prefix = os.path.join(staging_dir, 'warc') if remote_non_s3 else prefix_path + + configs = [ + dict( + shard_csv=shard_csv, + prefix_path=write_prefix, + subprefix=writer_subprefix, + writer_info=writer_info, + write_paths_as_metadata_records=write_paths_as_metadata_records, + log_every_n=log_every_n, + readers=readers_per_process, + aws_region_name=aws_region_name, + max_attempts=max_attempts, + max_file_size=max_file_size, + uvloop=uvloop, + warc_download_prefix=warc_download_prefix, + ) + for shard_csv in shard_csvs + ] + + logger.info('Launching %d worker processes x %d readers', len(configs), readers_per_process) + mp_context = multiprocessing.get_context() + file_sequence = mp_context.Value('i', 0) + with ProcessPoolExecutor( + max_workers=len(configs), mp_context=mp_context, initializer=_init_worker, initargs=(file_sequence,) + ) as ex: + results = list(ex.map(_run_worker, configs)) + total = sum(r for r in results if r and r > 0) + n_files = file_sequence.value + fetch_elapsed = time.time() - start + logger.info('All %d workers done: %d records in %d files in %.1fs (%.0f rec/s)', + len(configs), total, n_files, fetch_elapsed, total / fetch_elapsed if fetch_elapsed else 0) + + if remote_non_s3: + _copy_staged_files(write_prefix, prefix_path) + + logger.info('Repackage complete -> %d files %s (%.1fs total)', + n_files, _output_filename(prefix_path, sequence=1, subprefix=writer_subprefix), + time.time() - start) + return total + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +def _copy_staged_files(staging_prefix: str, prefix_path: str) -> List[str]: + """Copy the locally staged -... files to -... (e.g. on hf://).""" + staged = sorted(glob.glob(f'{staging_prefix}-*.warc.gz')) + + def _copy(src): + dest = prefix_path + src[len(staging_prefix):] + with open(src, 'rb') as fi, fsspec.open(dest, 'wb') as fo: + shutil.copyfileobj(fi, fo, length=8 * 1024 * 1024) + logger.info('Wrote %s', dest) + return dest + + with ThreadPoolExecutor(max_workers=max(1, min(_COPY_CONCURRENCY, len(staged)))) as ex: + return list(ex.map(_copy, staged)) diff --git a/cdx_toolkit/repackage/s3_utils.py b/cdx_toolkit/repackage/s3_utils.py new file mode 100644 index 0000000..ddd48f3 --- /dev/null +++ b/cdx_toolkit/repackage/s3_utils.py @@ -0,0 +1,62 @@ +import asyncio +import logging +from typing import Tuple +from os import urandom + +try: + from botocore.exceptions import ClientError, EndpointConnectionError +except ImportError: # pragma: no cover - S3 support is optional (cdx_toolkit[s3]) + class ClientError(Exception): + pass + + class EndpointConnectionError(Exception): + pass + + +logger = logging.getLogger(__name__) + + +def parse_s3_uri(uri: str) -> Tuple[str, str]: + """Parse a S3 URI and return bucket and prefix.""" + if not uri.startswith('s3://'): + raise ValueError(f'Not an S3 URI: {uri}') + rest = uri[5:] + i = rest.find('/') + if i <= 0 or i == len(rest) - 1: + raise ValueError(f'Malformed S3 URI: {uri}') + return rest[:i], rest[i+1:] + + +def is_s3_url(url: str) -> bool: + return url.startswith('s3:/') + + +async def with_retries(coro_factory, *, op_name: str, max_attempts: int, base_backoff_seconds: float): + """Perform operation with retries and backoff.""" + last_exc = None + for attempt in range(1, max_attempts + 1): + try: + return await coro_factory() + except (TimeoutError, ClientError, EndpointConnectionError) as exc: + last_exc = exc + if attempt >= max_attempts: + logger.error('%s failed after %d attempts: %r', op_name, attempt, exc) + break + sleep_s = _backoff(attempt, base_backoff_seconds) + logger.warning( + '%s failed (attempt %d/%d) - retrying in %.2fs', + op_name, + attempt, + max_attempts, + sleep_s, + ) + await asyncio.sleep(sleep_s) + raise last_exc + + +def _backoff(attempt: int, base_backoff_seconds: float) -> float: + """Time to sleep based on number of attempts""" + base = base_backoff_seconds * (2 ** (attempt - 1)) + + # Add random jitter between 80-120% of base delay + return max(0.05, base * (0.8 + 0.4 * urandom(1)[0] / 255)) diff --git a/cdx_toolkit/repackage/s3_writer.py b/cdx_toolkit/repackage/s3_writer.py new file mode 100644 index 0000000..124524d --- /dev/null +++ b/cdx_toolkit/repackage/s3_writer.py @@ -0,0 +1,172 @@ +import logging +from typing import List, Dict, Optional + +from cdx_toolkit.repackage.s3_utils import ( + with_retries, +) + +logger = logging.getLogger(__name__) + + +async def mpu_create( + s3, + bucket: str, + key: str, + *, + max_attempts: int, + base_backoff_seconds: float, +): + """Create multi part upload to S3.""" + kwargs = {'Bucket': bucket, 'Key': key} + resp = await with_retries( + lambda: s3.create_multipart_upload(**kwargs), + op_name=f'create_multipart_upload {bucket}/{key}', + max_attempts=max_attempts, + base_backoff_seconds=base_backoff_seconds, + ) + return resp['UploadId'] + + +async def mpu_upload_part( + s3, + bucket: str, + key: str, + upload_id: str, + part_number: int, + body: bytes, + max_attempts: int, + base_backoff_seconds: float, +) -> str: + """Upload a part of a multi-part upload to S3.""" + resp = await with_retries( + lambda: s3.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=part_number, + Body=body, + ), + op_name=f'upload_part {bucket}/{key}#{part_number}', + max_attempts=max_attempts, + base_backoff_seconds=base_backoff_seconds, + ) + return resp['ETag'] + + +async def mpu_complete( + s3, + bucket: str, + key: str, + upload_id: str, + parts: List[Dict], + max_attempts: int, + base_backoff_seconds: float, +): + """Send complete for multi-part upload.""" + await with_retries( + lambda: s3.complete_multipart_upload( + Bucket=bucket, Key=key, UploadId=upload_id, MultipartUpload={'Parts': parts} + ), + op_name=f'complete_multipart_upload {bucket}/{key}', + max_attempts=max_attempts, + base_backoff_seconds=base_backoff_seconds, + ) + + +async def mpu_abort(s3, bucket: str, key: str, upload_id: str): + """Abort mult-part upload.""" + try: + await s3.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + except Exception: + logger.exception('Failed to abort MPU %s on %s/%s', upload_id, bucket, key) + + +class S3ShardWriter: + """Manages one MPU: buffers bytes, uploads >=5 MiB parts, completes on close.""" + + def __init__( + self, + s3_client, + shard_key: str, + dest_bucket: str, + content_type: Optional[str], + min_part_size: int, + max_attempts: int, + base_backoff_seconds: float, + ): + self.s3_client = s3_client + self.shard_key = shard_key + self.dest_bucket = dest_bucket + self.content_type = content_type + self.min_part_size = min_part_size + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + self.upload_id: Optional[str] = None + self.part_number = 1 + self.parts: List[Dict] = [] + self.buffer = bytearray() + + async def start(self): + self.upload_id = await mpu_create( + self.s3_client, + self.dest_bucket, + self.shard_key, + max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + ) + logger.info('Started MPU for %s (UploadId=%s)', self.shard_key, self.upload_id) + + async def _flush_full_parts(self): + while len(self.buffer) >= self.min_part_size: + chunk = self.buffer[: self.min_part_size] + del self.buffer[: self.min_part_size] + etag = await mpu_upload_part( + self.s3_client, + self.dest_bucket, + self.shard_key, + self.upload_id, + self.part_number, + bytes(chunk), + self.max_attempts, + self.base_backoff_seconds, + ) + self.parts.append({'PartNumber': self.part_number, 'ETag': etag}) + self.part_number += 1 + + async def write(self, data: bytes): + self.buffer.extend(data) + await self._flush_full_parts() + + async def close(self): + try: + if self.buffer: + etag = await mpu_upload_part( + self.s3_client, + self.dest_bucket, + self.shard_key, + self.upload_id, + self.part_number, + bytes(self.buffer), + self.max_attempts, + self.base_backoff_seconds, + ) + self.parts.append({'PartNumber': self.part_number, 'ETag': etag}) + self.part_number += 1 + self.buffer.clear() + + if self.parts: + await mpu_complete( + self.s3_client, + self.dest_bucket, + self.shard_key, + self.upload_id, + self.parts, + self.max_attempts, + self.base_backoff_seconds, + ) + logger.info('Completed MPU for %s with %d parts.', self.shard_key, len(self.parts)) + except Exception: + logger.exception('Completing MPU failed for %s; attempting abort.', self.shard_key) + if self.upload_id: + await mpu_abort(self.s3_client, self.dest_bucket, self.shard_key, self.upload_id) + raise diff --git a/cdx_toolkit/repackage/sources/__init__.py b/cdx_toolkit/repackage/sources/__init__.py new file mode 100644 index 0000000..af3d689 --- /dev/null +++ b/cdx_toolkit/repackage/sources/__init__.py @@ -0,0 +1,3 @@ +from cdx_toolkit.repackage.sources.base import RangeJobSource, CostEstimate + +__all__ = ['RangeJobSource', 'CostEstimate'] diff --git a/cdx_toolkit/repackage/sources/athena.py b/cdx_toolkit/repackage/sources/athena.py new file mode 100644 index 0000000..2dfbaf6 --- /dev/null +++ b/cdx_toolkit/repackage/sources/athena.py @@ -0,0 +1,202 @@ +import itertools +import logging +import time +from typing import Iterator, Iterable, Optional + +from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.sources.base import RangeJobSource, CostEstimate +from cdx_toolkit.repackage.sources.sql_base import ( + validate_result_columns, + join_warc_url, + REQUIRED_RESULT_COLUMNS, +) + + +logger = logging.getLogger(__name__) + +# Athena pricing is ~$5 per TB scanned (used for the post-run cost estimate). +ATHENA_USD_PER_TB = 5.0 + + +class AthenaSource(RangeJobSource): + """RangeJobs from a query against the CC columnar index via AWS Athena.""" + + def __init__( + self, + *, + query: str, + database: str, + s3_output_location: str, + warc_download_prefix: Optional[str], + n_crawls: Optional[int] = None, + region_name: str = 'us-east-1', + max_wait_time: int = 300, + ): + self.query = query + self.database = database + self.s3_output_location = s3_output_location + self.warc_download_prefix = warc_download_prefix + self._n_crawls = n_crawls + self.region_name = region_name + self.max_wait_time = max_wait_time + + def estimate_cost(self) -> CostEstimate: + return CostEstimate(n_crawls=self._n_crawls, engine='athena') + + def _make_client(self): + import boto3 + from botocore.config import Config + + config = Config( + region_name=self.region_name, + read_timeout=60, + retries={'max_attempts': 3, 'mode': 'adaptive'}, + ) + return boto3.client('athena', config=config) + + def iter_rows(self) -> Iterator[dict]: + """Run the query and yield the result rows as dicts (column name -> string value).""" + client = self._make_client() + query_execution_id = run_athena_query( + client, self.query, self.database, self.s3_output_location, self.max_wait_time + ) + try: + yield from iter_result_rows(client, query_execution_id) + finally: + report_query_cost(client, query_execution_id) + + def iter_range_jobs(self) -> Iterator[RangeJob]: + rows = self.iter_rows() + first = next(rows, None) + if first is not None: + validate_result_columns(list(first)) + for row in itertools.chain([first], rows): + yield row_to_range_job(row, self.warc_download_prefix) + + +def run_athena_query(client, query: str, database: str, s3_output_location: str, max_wait_time: int = 300) -> str: + """Start an Athena query and block until it completes; return the execution id. + + Raises if the query does not reach the SUCCEEDED state. If the wait is + interrupted (Ctrl-C / cancellation) or times out, the query is cancelled + server-side so we don't keep paying for a scan whose results we'll never read.""" + logger.info('Executing Athena query: %s', query) + + response = client.start_query_execution( + QueryString=query, + QueryExecutionContext={'Database': database}, + ResultConfiguration={'OutputLocation': s3_output_location}, + ) + + query_execution_id = response['QueryExecutionId'] + logger.info('Query execution started. ID: %s', query_execution_id) + + try: + status = _wait_for_query_completion(client, query_execution_id, max_wait_time) + except BaseException: + # Ctrl-C, asyncio cancellation, or timeout: stop the query server-side + # to bound the Athena scan cost, then propagate the original exception. + _stop_query(client, query_execution_id) + raise + + if status != 'SUCCEEDED': + raise Exception(f'Query failed with status: {status}') + + return query_execution_id + + +def _stop_query(client, query_execution_id: str) -> None: + """Best-effort cancellation of a running Athena query.""" + logger.warning('Cancelling Athena query %s ...', query_execution_id) + try: + client.stop_query_execution(QueryExecutionId=query_execution_id) + logger.warning('Athena query %s cancelled', query_execution_id) + except Exception as e: # pragma: no cover - best-effort cleanup + logger.warning('Failed to cancel Athena query %s: %r', query_execution_id, e) + + +def report_query_cost(client, query_execution_id: str) -> None: + """Log the bytes scanned and an estimated USD cost for a completed query.""" + try: + response = client.get_query_execution(QueryExecutionId=query_execution_id) + scanned = response['QueryExecution'].get('Statistics', {}).get('DataScannedInBytes') + except Exception as e: # pragma: no cover - best-effort reporting + logger.debug('unable to read Athena query statistics: %r', e) + return + + if scanned is None: + return + + gb = scanned / 1e9 + usd = scanned / 1e12 * ATHENA_USD_PER_TB + logger.info('Athena scanned %.2f GB, estimated cost ~$%.4f', gb, usd) + + +def _wait_for_query_completion(client, query_execution_id: str, max_wait_time: int) -> str: + """Wait for query to complete and return final status""" + start_time = time.time() + + while time.time() - start_time < max_wait_time: + response = client.get_query_execution(QueryExecutionId=query_execution_id) + + status = response['QueryExecution']['Status']['State'] + logger.info(f'Query status: {status}') + + if status in ['SUCCEEDED', 'FAILED', 'CANCELLED']: + if status == 'FAILED': + error_reason = response['QueryExecution']['Status'].get('StateChangeReason', 'Unknown error') + logger.info(f'Query failed: {error_reason}') + return status + + time.sleep(2) + + raise TimeoutError(f'Query did not complete within {max_wait_time} seconds') + + +def iter_result_rows(client, query_execution_id: str) -> Iterable[dict]: + """Retrieve query results as dicts (column name -> string value)""" + paginator = client.get_paginator('get_query_results') + page_iterator = paginator.paginate(QueryExecutionId=query_execution_id) + column_names = None + + for page in page_iterator: + rows = page['ResultSet']['Rows'] + + # Get column names from first page + if column_names is None and rows: + column_names = [col['VarCharValue'] for col in rows[0]['Data']] + rows = rows[1:] # Skip header row + + # Process data rows + for row in rows: + row_data = [cell.get('VarCharValue', None) for cell in row['Data']] + yield dict(zip(column_names, row_data)) + + +def row_to_range_job(row: dict, warc_download_prefix: Optional[str]) -> RangeJob: + warc_filename = row['warc_filename'] + extra = {k: v for k, v in row.items() if k not in REQUIRED_RESULT_COLUMNS} + return RangeJob( + url=join_warc_url(warc_download_prefix, warc_filename), + offset=int(row['warc_record_offset']), + length=int(row['warc_record_length']), + filename=warc_filename, + extra=extra or None, + ) + + +def iter_range_jobs(client, query_execution_id: str, warc_download_prefix: Optional[str]) -> Iterable[RangeJob]: + """Retrieve query results and convert each row to a RangeJob""" + rows = iter_result_rows(client, query_execution_id) + first = next(rows, None) + if first is None: + return + validate_result_columns(list(first)) + for row in itertools.chain([first], rows): + yield row_to_range_job(row, warc_download_prefix) + + +def get_databases(client) -> list: + """Get list of available databases""" + response = client.list_databases(CatalogName='AwsDataCatalog') + return [db['Name'] for db in response['DatabaseList']] diff --git a/cdx_toolkit/repackage/sources/base.py b/cdx_toolkit/repackage/sources/base.py new file mode 100644 index 0000000..e253cd7 --- /dev/null +++ b/cdx_toolkit/repackage/sources/base.py @@ -0,0 +1,35 @@ +from abc import ABC, abstractmethod +from typing import Iterator, NamedTuple, Optional + +from cdx_toolkit.repackage.data_classes import RangeJob + + +class CostEstimate(NamedTuple): + """Describes the scan a source is about to run, for the cost-confirmation guard. + + n_crawls is the number of crawls the scan is bounded to, or None when the scan + is unbounded / its pruning cannot be verified (e.g. raw SQL, all-crawls glob).""" + + n_crawls: Optional[int] + engine: str + + +class RangeJobSource(ABC): + """A source of RangeJobs for the repackage pipeline. + + A source owns its own stage-1 resource (Athena client, DuckDB connection, or an + fsspec file handle) and yields RangeJobs synchronously; the pipeline orchestrator + bridges the sync generator into the async fetch/write stages, and owns queueing, + the record limit, counting, and stop-sentinel emission.""" + + def estimate_cost(self) -> Optional[CostEstimate]: + """Return a CostEstimate for the cost guard, or None for sources that never + incur a per-scan charge (cdx files, csv).""" + return None + + @abstractmethod + def iter_range_jobs(self) -> Iterator[RangeJob]: + """Yield RangeJobs. Implementations open their own client/connection lazily + and close it in a finally. Each RangeJob carries `url` (authoritative for + fetching) and, where known, the relative `filename`.""" + raise NotImplementedError diff --git a/cdx_toolkit/repackage/sources/csv.py b/cdx_toolkit/repackage/sources/csv.py new file mode 100644 index 0000000..3c4f270 --- /dev/null +++ b/cdx_toolkit/repackage/sources/csv.py @@ -0,0 +1,120 @@ +import csv +import logging +from typing import Iterator, Optional + +import fsspec + +from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.sources.base import RangeJobSource +from cdx_toolkit.repackage.sources.sql_base import join_warc_url + + +logger = logging.getLogger(__name__) + +# Fetch-job columns name the WARC *location* (mirroring the columnar index's column +# names) so they never collide with extra analysis columns a raw --query may SELECT +# -- e.g. the index's own `url` (the captured page URL). `warc_filename` is a filename +# relative to the WARC download prefix, or a full URL (s3://, https://, hf://...). +FILENAME_FIELDS = ['warc_filename', 'warc_record_offset', 'warc_record_length'] + +# Columns the reader interprets as the fetch job; everything else round-trips as extra. +_KNOWN_FIELDS = set(FILENAME_FIELDS) + + +class RangeJobCsvWriter: + """Write RangeJobs to a CSV (local or remote via fsspec). + + The `warc_filename` column holds the full download URL of each job, so the CSV can + be read without knowing the WARC download prefix. Any extra columns carried on a + RangeJob (from a raw SQL --query) are appended after the fetch-job columns. The + header is written lazily from the first job so its extra columns can be discovered.""" + + def __init__(self, path: str): + self.path = path + self._ctx = fsspec.open(path, 'wt', newline='') + self._fh = self._ctx.__enter__() + self._writer = None + + def _init_writer(self, fieldnames) -> None: + self._writer = csv.DictWriter( + self._fh, fieldnames=fieldnames, extrasaction='ignore', restval='' + ) + self._writer.writeheader() + + def write(self, job: RangeJob) -> None: + row = { + 'warc_filename': job.url, + 'warc_record_offset': job.offset, + 'warc_record_length': job.length, + } + if job.extra: + row.update(job.extra) + + if self._writer is None: + extra_fields = [k for k in (job.extra or {}) if k not in FILENAME_FIELDS] + self._init_writer(FILENAME_FIELDS + extra_fields) + self._writer.writerow(row) + + def close(self) -> None: + if self._ctx is not None: + # No rows written: still emit the base header for a valid (empty) file. + if self._writer is None: + self._init_writer(FILENAME_FIELDS) + self._ctx.__exit__(None, None, None) + self._ctx = None + self._fh = None + + +class CsvSource(RangeJobSource): + """RangeJobs read from a CSV/TSV with `warc_filename`, `warc_record_offset` and + `warc_record_length` columns. + + A `warc_filename` that is a full URL (s3://, https://, hf://...) is used as-is, + otherwise the WARC download prefix is prepended. Any other columns round-trip onto + RangeJob.extra. TSV is detected from a `.tsv`/`.tsv.gz` extension; `.gz` inputs are + decompressed. + + With `sort` (the default), jobs are sorted by (WARC file, record offset) before + being emitted -- grouping same-file records with ascending offsets for better S3 + range-read locality. This buffers all rows in memory; pass sort=False to stream + an already-sorted file without buffering.""" + + def __init__(self, path: str, warc_download_prefix: Optional[str], sort: bool = True): + self.path = path + self.warc_download_prefix = warc_download_prefix + self.sort = sort + + def iter_range_jobs(self) -> Iterator[RangeJob]: + path = str(self.path) + delimiter = '\t' if path.endswith(('.tsv', '.tsv.gz')) else ',' + compression = 'gzip' if path.endswith('.gz') else None + + buffered = [] if self.sort else None + with fsspec.open(self.path, 'rt', newline='', compression=compression) as fh: + reader = csv.DictReader(fh, delimiter=delimiter) + missing = [f for f in FILENAME_FIELDS if f not in (reader.fieldnames or [])] + if missing: + raise ValueError( + f'range-jobs CSV {self.path} must have the columns {", ".join(FILENAME_FIELDS)} ' + f'(got header: {reader.fieldnames})' + ) + + for row in reader: + filename = row['warc_filename'] + extra = {k: v for k, v in row.items() if k not in _KNOWN_FIELDS} + job = RangeJob( + url=join_warc_url(self.warc_download_prefix, filename), + offset=int(row['warc_record_offset']), + length=int(row['warc_record_length']), + filename=filename, + extra=extra or None, + ) + if buffered is None: + yield job + else: + buffered.append(job) + + if buffered is not None: + buffered.sort(key=lambda j: (j.filename or j.url or '', j.offset)) + for job in buffered: + yield job diff --git a/cdx_toolkit/repackage/sources/duckdb.py b/cdx_toolkit/repackage/sources/duckdb.py new file mode 100644 index 0000000..86c0992 --- /dev/null +++ b/cdx_toolkit/repackage/sources/duckdb.py @@ -0,0 +1,139 @@ +import logging +from typing import Iterator, List, Optional + +from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.sources.base import RangeJobSource, CostEstimate +from cdx_toolkit.repackage.sources.sql_base import ( + build_sql, + validate_result_columns, + join_warc_url, + REQUIRED_RESULT_COLUMNS, +) + + +logger = logging.getLogger(__name__) + +try: + import duckdb + _HAS_DUCKDB = True +except ImportError: # pragma: no cover - exercised in minimal installs + duckdb = None + _HAS_DUCKDB = False + + +def _build_from_clause(index_path: str, crawls: Optional[List[str]]) -> str: + """Build a read_parquet(...) FROM expression over the CC columnar index. + + When crawls are given we glob only those crawl partitions (the pruning lever); + otherwise we glob every crawl (expensive -> the cost guard fires).""" + base = index_path.rstrip('/') + if crawls: + globs = [f"'{base}/crawl={c}/subset=warc/*.parquet'" for c in crawls] + else: + globs = [f"'{base}/crawl=*/subset=warc/*.parquet'"] + return f"read_parquet([{', '.join(globs)}], hive_partitioning=true)" + + +class DuckDbSource(RangeJobSource): + """RangeJobs from a query against the CC columnar index via DuckDB (read_parquet on S3). + + Reads the public CommonCrawl parquet directly; AWS region/credentials come from + the environment (the public bucket is readable with valid credentials).""" + + def __init__( + self, + *, + query: Optional[str] = None, + hostnames: Optional[List[str]] = None, + domains: Optional[List[str]] = None, + crawls: Optional[List[str]] = None, + index_path: str, + warc_download_prefix: Optional[str], + limit: int = 0, + sort: bool = True, + region_name: str = 'us-east-1', + ): + self.raw_query = query + self.hostnames = hostnames + self.domains = domains + self.crawls = crawls + self.index_path = index_path + self.warc_download_prefix = warc_download_prefix + self.limit = limit + self.sort = sort + self.region_name = region_name + + def estimate_cost(self) -> CostEstimate: + if self.raw_query is not None: + return CostEstimate(n_crawls=None, engine='duckdb') + return CostEstimate(n_crawls=len(self.crawls) if self.crawls else None, engine='duckdb') + + def _build_query(self) -> str: + if self.raw_query is not None: + return self.raw_query + from_clause = _build_from_clause(self.index_path, self.crawls) + # crawl pruning is done in the FROM glob, so no crawl IN (...) in the WHERE + return build_sql( + from_clause, self.hostnames, crawls=None, limit=self.limit, + url_host_registered_domains=self.domains, order_by=self.sort, + ) + + def iter_rows(self, validate: bool = False) -> Iterator[dict]: + """Run the query and yield the result rows as dicts (column name -> value).""" + if not _HAS_DUCKDB: + raise RuntimeError( + 'DuckDB engine requires optional dependencies. Install cdx_toolkit[duckdb].' + ) + + query = self._build_query() + logger.info('Executing DuckDB query: %s', query) + + con = duckdb.connect() + try: + con.execute('INSTALL httpfs; LOAD httpfs;') + con.execute(f"SET s3_region='{self.region_name}';") + # CC timestamps are UTC (timestamp literals in queries, timestamps in the results) + con.execute("SET TimeZone='UTC';") + try: + # find AWS credentials like boto3 does (env, ~/.aws profiles, SSO, instance roles) + con.execute('INSTALL aws; LOAD aws;') + con.execute(f"CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION '{self.region_name}');") + except Exception as e: # pragma: no cover - no credentials or aws extension unavailable + logger.debug('duckdb AWS credential chain not used: %r', e) + # Be resilient to transient S3 read timeouts on large parquet partitions. + for stmt in ('SET http_timeout=120000;', 'SET http_retries=5;'): + try: + con.execute(stmt) + except Exception as e: # pragma: no cover - setting unsupported on this duckdb + logger.debug('duckdb setting skipped: %s (%r)', stmt, e) + if 'hf://buckets/' in query: + # DuckDB cannot read Hugging Face Storage Buckets itself; use their fsspec filesystem. + from huggingface_hub import HfFileSystem + + con.register_filesystem(HfFileSystem()) + cur = con.execute(query) + + col_names = [d[0] for d in cur.description] + if validate: + validate_result_columns(col_names) + + while True: + rows = cur.fetchmany(1000) + if not rows: + break + for row in rows: + yield dict(zip(col_names, row)) + finally: + con.close() + + def iter_range_jobs(self) -> Iterator[RangeJob]: + for row in self.iter_rows(validate=True): + warc_filename = row['warc_filename'] + extra = {name: value for name, value in row.items() if name not in REQUIRED_RESULT_COLUMNS} + yield RangeJob( + url=join_warc_url(self.warc_download_prefix, warc_filename), + offset=int(row['warc_record_offset']), + length=int(row['warc_record_length']), + filename=warc_filename, + extra=extra or None, + ) diff --git a/cdx_toolkit/repackage/sources/records.py b/cdx_toolkit/repackage/sources/records.py new file mode 100644 index 0000000..518eaef --- /dev/null +++ b/cdx_toolkit/repackage/sources/records.py @@ -0,0 +1,45 @@ +import logging +from typing import Iterable, Iterator, Mapping, Optional + +from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.sources.base import RangeJobSource +from cdx_toolkit.repackage.sources.sql_base import join_warc_url + + +logger = logging.getLogger(__name__) + +WARC_LOCATION_FIELDS = ('filename', 'offset', 'length') + + +class MissingWarcLocation(ValueError): + """A record has no WARC location (filename, offset, length), e.g. an Internet Archive capture.""" + + +class RecordsSource(RangeJobSource): + """RangeJobs from index records (as written by `cdxt index`) with filename, offset and length fields. + + With `sort` (the default), jobs are sorted by (WARC file, record offset) for better + range-read locality. The records are then read (and validated) when the source is + created, and kept in memory; pass sort=False to stream them in input order.""" + + def __init__(self, records: Iterable[Mapping], warc_download_prefix: Optional[str], sort: bool = True): + self.warc_download_prefix = warc_download_prefix + self.jobs = (self.to_range_job(record) for record in records) + if sort: + self.jobs = sorted(self.jobs, key=lambda j: (j.filename or j.url, j.offset)) + + def iter_range_jobs(self) -> Iterator[RangeJob]: + yield from self.jobs + + def to_range_job(self, record: Mapping) -> RangeJob: + if not all(record.get(field) not in (None, '') for field in WARC_LOCATION_FIELDS): + raise MissingWarcLocation( + f'capture {record.get("url")} has no WARC location (filename, offset, length); ' + 'use --wb to fetch it from a wayback machine' + ) + return RangeJob( + url=join_warc_url(self.warc_download_prefix, record['filename']), + offset=int(record['offset']), + length=int(record['length']), + filename=record['filename'], + ) diff --git a/cdx_toolkit/repackage/sources/sql_base.py b/cdx_toolkit/repackage/sources/sql_base.py new file mode 100644 index 0000000..5724de0 --- /dev/null +++ b/cdx_toolkit/repackage/sources/sql_base.py @@ -0,0 +1,199 @@ +import logging +import re +from typing import List, Optional + +from cdx_toolkit.commoncrawl import normalize_crawl, get_cc_endpoints, match_cc_crawls + + +logger = logging.getLogger(__name__) + +# Required output columns that any index query (built or raw) must provide. +REQUIRED_RESULT_COLUMNS = ('warc_filename', 'warc_record_offset', 'warc_record_length') + +# Hostnames, TLDs and crawl names only ever contain these characters. Validating +# against this set both prevents SQL injection and catches malformed input early. +_SQL_LITERAL_RE = re.compile(r'^[A-Za-z0-9.\-]+$') + +# Default Common Crawl index mirror used to resolve --crawl to concrete crawl names. +CC_INDEX_MIRROR = 'https://index.commoncrawl.org/' + + +def escape_sql_literal(value: str) -> str: + """Validate and quote a value for safe inclusion in a SQL string literal. + + Only hostname/TLD/crawl-name characters (letters, digits, dot, hyphen) are + allowed, so the result cannot break out of the quotes or inject SQL.""" + if not isinstance(value, str) or not _SQL_LITERAL_RE.match(value): + raise ValueError( + f'Invalid value for SQL query literal: {value!r} ' + '(allowed characters: letters, digits, dot, hyphen)' + ) + return "'" + value + "'" + + +def quote_sql_string(value: str) -> str: + """Quote any string for a SQL string literal (single quotes are doubled).""" + if any(ord(c) < 32 for c in value): + raise ValueError(f'Invalid value for SQL query literal: {value!r} (control characters)') + return "'" + value.replace("'", "''") + "'" + + +def sql_like_prefix(prefix: str) -> str: + """LIKE pattern (with ESCAPE '\\') matching all strings that start with `prefix`.""" + escaped = prefix.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_') + return f"LIKE {quote_sql_string(escaped + '%')} ESCAPE '\\'" + + +def surt_tld(surt: str) -> str: + """'com,example)/path' -> 'com'""" + return surt.split(',', 1)[0].split(')', 1)[0] + + +def build_where_sql( + url_host_names: Optional[List[str]] = None, + crawls: Optional[List[str]] = None, + url_host_registered_domains: Optional[List[str]] = None, + conditions: Optional[List[str]] = None, + url_surtkey_prefixes: Optional[List[str]] = None, + url_surtkeys: Optional[List[str]] = None, +) -> str: + """Build the WHERE body (without the `WHERE` keyword) shared by all SQL engines. + + The guided filter matches on `url_host_name` (exact host, e.g. www.example.com), + `url_host_registered_domain` (e.g. example.com, which also covers its subdomains) + and/or `url_surtkey` (SURT keys of URL patterns: `url_surtkey_prefixes` match all + keys starting with them, `url_surtkeys` exactly); predicates are OR-ed together. + At least one host, domain or SURT key is required. If `crawls` is a non-empty list + of crawl names (e.g. ['CC-MAIN-2025-33']), a `crawl IN (...)` partition filter is + added -- the main lever for reducing scan cost. `conditions` are further (already + escaped) predicates that are AND-ed. Engines differ only in their FROM clause (see + build_sql).""" + url_host_names = url_host_names or [] + domains = url_host_registered_domains or [] + surt_prefixes = url_surtkey_prefixes or [] + surts = url_surtkeys or [] + if not url_host_names and not domains and not surt_prefixes and not surts: + raise ValueError('an index query requires at least one hostname, registered domain or URL') + + tlds = {v.split('.')[-1] for v in (list(url_host_names) + list(domains))} + tlds |= {surt_tld(s) for s in surt_prefixes + surts} + query_tlds = ' OR '.join(f'url_host_tld = {escape_sql_literal(t)}' for t in sorted(tlds)) + + host_predicates = [f'url_host_name = {escape_sql_literal(h)}' for h in url_host_names] + host_predicates += [f'url_host_registered_domain = {escape_sql_literal(d)}' for d in domains] + host_predicates += [f'url_surtkey = {quote_sql_string(s)}' for s in surts] + host_predicates += [f'url_surtkey {sql_like_prefix(s)}' for s in surt_prefixes] + query_hosts = ' OR '.join(host_predicates) + + clauses = [ + "subset = 'warc'", + f'({query_tlds}) -- help the query optimizer', + f'({query_hosts})', + ] + if crawls: + crawl_in = ', '.join(escape_sql_literal(c) for c in crawls) + clauses.append(f'crawl IN ({crawl_in})') + clauses.extend(conditions or []) + + return '\n AND '.join(clauses) + + +# Sorting range jobs by (warc_filename, warc_record_offset) groups records of the +# same WARC file with ascending offsets, which improves S3 range-read locality (and +# enables coalescing adjacent ranges) at fetch time. +ORDER_BY_SQL = 'ORDER BY warc_filename, warc_record_offset' + + +def build_sql( + from_clause: str, + url_host_names: Optional[List[str]] = None, + crawls: Optional[List[str]] = None, + limit: int = 0, + url_host_registered_domains: Optional[List[str]] = None, + order_by: bool = True, + columns: Optional[List[str]] = None, + conditions: Optional[List[str]] = None, + url_surtkey_prefixes: Optional[List[str]] = None, + url_surtkeys: Optional[List[str]] = None, +) -> str: + """Assemble a full SELECT for the columnar index. + + `from_clause` is the text following FROM (e.g. `ccindex` for Athena, or a + `read_parquet(...)` expression for DuckDB). When `order_by` is set, results are + sorted by (warc_filename, warc_record_offset) for fetch-time read locality. + `columns` defaults to the REQUIRED_RESULT_COLUMNS.""" + where_sql = build_where_sql( + url_host_names, crawls, url_host_registered_domains=url_host_registered_domains, conditions=conditions, + url_surtkey_prefixes=url_surtkey_prefixes, url_surtkeys=url_surtkeys, + ) + order_sql = f'\n {ORDER_BY_SQL}' if order_by else '' + limit_sql = f'\n LIMIT {limit}' if limit and limit > 0 else '' + columns_sql = ', '.join(columns or REQUIRED_RESULT_COLUMNS) + + return f""" + SELECT + {columns_sql} + FROM {from_clause} + WHERE {where_sql}{order_sql}{limit_sql}""" + + +def build_athena_query( + url_host_names: Optional[List[str]] = None, + crawls: Optional[List[str]] = None, + limit: int = 0, + table: str = 'ccindex', + url_host_registered_domains: Optional[List[str]] = None, + order_by: bool = True, +) -> str: + """Athena flavour of build_sql (FROM ).""" + return build_sql( + table, url_host_names, crawls=crawls, limit=limit, + url_host_registered_domains=url_host_registered_domains, + order_by=order_by, + ) + + +def validate_result_columns(column_names) -> None: + """Raise a clear error if the query result lacks the required columns.""" + missing = [c for c in REQUIRED_RESULT_COLUMNS if c not in (column_names or [])] + if missing: + raise ValueError( + 'Index query result is missing required columns: ' + ', '.join(missing) + + '. The query (including a raw --query) must SELECT ' + + ', '.join(REQUIRED_RESULT_COLUMNS) + '.' + ) + + +def join_warc_url(prefix: Optional[str], warc_filename: str) -> str: + """Join a download prefix with a warc_filename robustly. + + - if warc_filename is already absolute (contains '://'), return it unchanged + (supports custom queries / self-contained CSVs whose value is a full URL); + - if prefix is empty/None, return warc_filename unchanged; + - otherwise join with exactly one '/' (no double slash, no missing slash).""" + if '://' in warc_filename: + return warc_filename + if not prefix: + return warc_filename + return prefix.rstrip('/') + '/' + warc_filename.lstrip('/') + + +def endpoint_to_crawl_name(endpoint: str) -> str: + """Turn a collinfo cdx-api endpoint into its crawl name. + + e.g. 'https://index.commoncrawl.org/CC-MAIN-2025-33-index' -> 'CC-MAIN-2025-33'.""" + name = endpoint.rstrip('/').split('/')[-1] + if name.endswith('-index'): + name = name[:-len('-index')] + return name + + +def resolve_crawl_names(crawl_arg) -> List[str]: + """Resolve a --crawl value to concrete CC-MAIN crawl names for the partition filter. + + Reuses the CDX path's helpers so `--crawl` accepts the same forms (comma-separated + names or an integer for the most recent N crawls).""" + crawls = normalize_crawl([crawl_arg]) + raw_index_list = get_cc_endpoints(CC_INDEX_MIRROR) + matched = match_cc_crawls(crawls, raw_index_list) + return [endpoint_to_crawl_name(ep) for ep in matched] diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py new file mode 100644 index 0000000..f613e42 --- /dev/null +++ b/cdx_toolkit/repackage/warc_filter.py @@ -0,0 +1,599 @@ +import asyncio +import contextlib +import logging +import os +import statistics +import sys +from typing import Callable, List, Optional, Dict + + +from cdx_toolkit.repackage.s3_utils import ( + is_s3_url, +) +from cdx_toolkit.repackage.hf_utils import is_hf_url, make_hf_reader +from cdx_toolkit.repackage.data_classes import RangeJob, RangePayload, ThroughputTracker +from cdx_toolkit.repackage.warc_utils import create_new_writer_with_header +from cdx_toolkit.repackage.sources.base import RangeJobSource +from cdx_toolkit.repackage.warc_utils import get_bytes_from_warc_record, get_metadata_record_from_path + + +_STOP = object() + +logger = logging.getLogger(__name__) + + +class WARCFilter: + """Filter or extract specific records from WARC files based on CDX indexes. + + The WARC filter uses a three stage listner-producer-consumer pattern. + + Filter targets: + - CDX index files from local or remote file system, containing paths to WARC files and positions of target records. + + WARC reader: + - HTTP range reads + - S3 range reads + + WARC writer: + - Local file system + - S3 using multi-part uploads + """ + + def __init__( + self, + prefix_path: str, + writer_info: Dict, + source: RangeJobSource, + writer_subprefix: Optional[str] = None, + write_paths_as_metadata_records: Optional[List[str]] = None, + log_every_n: int = 1000, + warc_download_prefix: Optional[str] = None, + n_parallel_readers: int = 1, + max_attempts: int = 5, + base_backoff_seconds: float = 0.5, + # writer_kwargs: Optional[Dict] = None, + range_jobs_queue_size: int = 1000, + warc_records_queue_size: int = 200, + aws_region_name: str = 'us-east-1', + warc_version: str = '1.0', + content_type: Optional[str] = None, + min_part_size: int = 5 * 1024 * 1024, # 5 MiB (for upload) + max_file_size: Optional[int] = 1 * 1024 * 1024 * 1024, # 1 GiB (for WARC outputs) + next_file_sequence: Optional[Callable[[int], int]] = None, + ): + """Initialize the WARC filter. + + Args: + source: RangeJobSource that yields the WARC ranges to repackage. + prefix_path: Output path prefix for filtered WARC files. + writer_info: Dictionary containing writer metadata. + writer_subprefix: Optional subprefix for writer output paths. + write_paths_as_metadata_records: Optional list of file paths to write as metadata records. + log_every_n: Log progress every N records. + warc_download_prefix: Optional prefix to prepend to WARC URLs. + n_parallel_readers: Number of async readers per process (one writer per process). + max_attempts: Maximum retry attempts for failed operations. + base_backoff_seconds: Base backoff time in seconds for retries. + writer_kwargs: Optional additional kwargs for writers. + range_jobs_queue_size: Maximum size of range jobs queue. + warc_records_queue_size: Maximum size of WARC records queue. + aws_region_name: AWS region name for S3 operations. + warc_version: WARC format version (e.g., '1.0' or '1.1'). + content_type: Optional content type for WARC output. + min_part_size: Minimum part byte size for multipart uploads (default: 5 MiB). + max_file_size: Maximum byte size for individual WARC files (default: 1 GiB). + next_file_sequence: Number of the next output file, given the current one + (default: current + 1). + """ + self.source = source + self.prefix_path = prefix_path + self.writer_info = writer_info + self.writer_subprefix = writer_subprefix + self.write_paths_as_metadata_records = write_paths_as_metadata_records + self.log_every_n = log_every_n + self.warc_download_prefix = warc_download_prefix + + # self.writer_kwargs = writer_kwargs + self.range_jobs_queue_size = range_jobs_queue_size + self.warc_records_queue_size = warc_records_queue_size + self.aws_region_name = aws_region_name + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + + # Many async readers feed a single writer. Writing is a cheap verbatim byte copy + # (+ >=5 MiB MPU parts) that one coroutine sustains easily; multi-core scaling and + # output sharding are handled by running multiple *processes* (see + # cdx_toolkit/repackage/multiprocess.py), not multiple writers on one event loop. + self.num_readers = n_parallel_readers + + self.gzip = True + + self.warc_version = warc_version + self.content_type = content_type + self.min_part_size = min_part_size + self.max_file_size = max_file_size + # Number of the next output file, given the current one (0 before the first file). The + # multi-process mode shares one counter across processes, so file names are final right away. + self.next_file_sequence = next_file_sequence or (lambda sequence: sequence + 1) + + def filter(self) -> int: + """Perform the filtering process (calls async method via asyncio.run). + + Returns: + int: Number of records written, or -1 if interrupted. + """ + runner = asyncio.run + if os.environ.get('CDXT_UVLOOP') == '1': + try: + import uvloop + + runner = uvloop.run + logger.info('Using uvloop event loop') + except ImportError: + logger.warning('CDXT_UVLOOP=1 but uvloop is not installed; using default asyncio loop') + try: + return runner(self.filter_async()) + except KeyboardInterrupt: + logger.warning('Interrupted by user.') + + return -1 + + def needs_aws(self) -> bool: + """Returns true if the read/write (stage 2/3) S3 clients are needed. + + Sources own their own stage-1 resource (Athena client / DuckDB connection / + fsspec), so this only concerns WARC reads and output writes. + """ + return is_s3_url(self.warc_download_prefix) or is_s3_url(self.prefix_path) + + def get_boto3_base_config(self) -> Dict: + """Get boto3 base configuration for AWS client. + + Returns: + Dict: Boto3 base configuration object with retry and timeout settings. + """ + # Calculate max connections based on parallelism + # Each reader + writer needs connections, plus some overhead for retries + # max_pool_connections = max(50, (self.num_readers + self.num_writers) * 2) + + return dict( + region_name=self.aws_region_name, + retries={ + 'max_attempts': max(2, self.max_attempts), + 'mode': 'adaptive', # Better than 'standard' for variable workloads + }, + ) + + async def get_aws_clients(self) -> Optional[Dict]: + """Return async S3 clients for WARC reads/writes if needed. + + Stage-1 clients/connections are owned by the source, so this only builds the + read/write S3 clients used to fetch WARC ranges and write output. + + Raises: + SystemExit: If S3 is needed but Python version is < 3.9. + """ + if self.needs_aws(): + if sys.version_info.major < 3 or (sys.version_info.major >= 3 and sys.version_info.minor < 9): + logger.error('Reading and writing to S3 requires Python version >= 3.9') + sys.exit(1) + + import aioboto3 + from botocore.config import Config + + session = aioboto3.Session() + + # High-throughput config for range reads + read_config = Config( + max_pool_connections=self.num_readers * 3, + read_timeout=300, + tcp_keepalive=True, + **self.get_boto3_base_config(), + ) + + # Optimized config for multipart uploads (single writer) + write_config = Config( + max_pool_connections=8, + read_timeout=120, + connect_timeout=10, + **self.get_boto3_base_config(), + ) + + return { + 'read': session.client('s3', config=read_config), + 'write': session.client('s3', config=write_config), + } + else: + return None + + async def filter_async(self) -> int: + """Filter process using a three stage approach (job generator, warc reader, warc writer). + + Returns: + int: Number of records written. + """ + range_jobs_queue: asyncio.Queue = asyncio.Queue(maxsize=self.range_jobs_queue_size) + warc_records_queue: asyncio.Queue = asyncio.Queue(maxsize=self.warc_records_queue_size) + + read_is_hf = is_hf_url(self.warc_download_prefix) + + async with contextlib.AsyncExitStack() as stack: + read_s3_client = write_s3_client = None + # S3 read and/or write clients (built together when either side is S3). + if self.needs_aws(): + clients = await self.get_aws_clients() + read_s3_client = await stack.enter_async_context(clients['read']) + write_s3_client = await stack.enter_async_context(clients['write']) + + # HF reader for hf:// downloads (CDN ranged GETs, one resolve per WARC file). + hf_reader = None + if read_is_hf: + hf_reader = await stack.enter_async_context(make_hf_reader()) + + return await self._run_filter_pipeline( + range_jobs_queue=range_jobs_queue, + warc_records_queue=warc_records_queue, + read_s3_client=read_s3_client, + write_s3_client=write_s3_client, + hf_reader=hf_reader, + ) + + async def _produce_range_jobs(self, range_jobs_queue: asyncio.Queue) -> int: + """Drive the (sync) source in a worker thread, feeding the async queue. + + Owns counting and emitting one _STOP sentinel per reader in a finally -- so + readers never hang even if the source raises mid-iteration.""" + loop = asyncio.get_running_loop() + count = 0 + + def drain() -> int: + nonlocal count + for job in self.source.iter_range_jobs(): + asyncio.run_coroutine_threadsafe(range_jobs_queue.put(job), loop).result() + count += 1 + return count + + try: + await asyncio.to_thread(drain) + finally: + for _ in range(self.num_readers): + await range_jobs_queue.put(_STOP) + + logger.info('Generated %d range jobs', count) + return count + + async def _run_filter_pipeline( + self, + range_jobs_queue: asyncio.Queue, + warc_records_queue: asyncio.Queue, + read_s3_client=None, + write_s3_client=None, + hf_reader=None, + ) -> int: + """Run the actual filter pipeline with or without S3 client. + + Args: + range_jobs_queue: Queue for range jobs from the source. + warc_records_queue: Queue for WARC record payloads. + read_s3_client: Optional S3 client for reads from S3. + write_s3_client: Optional S3 client for writes S3. + hf_reader: Optional HF reader for hf:// downloads. + + Returns: + int: Number of records written. + """ + logger.info('Starting job generator, %d WARC readers, 1 WARC writer', self.num_readers) + + # Generate range jobs from the configured source (bridged sync->async in a thread). + job_generators = asyncio.create_task(self._produce_range_jobs(range_jobs_queue)) + + # Read WARC records based on file paths and ranges + warc_readers = [ + asyncio.create_task( + self.read_warc_records( + reader_id=i, + range_jobs_queue=range_jobs_queue, + warc_records_queue=warc_records_queue, + s3_client=read_s3_client, + hf_reader=hf_reader, + ) + ) + for i in range(self.num_readers) + ] + + # Write WARC records (a single writer owns the output shard for this process) + warc_writer = asyncio.create_task( + self.write_warc_records( + warc_records_queue=warc_records_queue, + s3_client=write_s3_client, + ) + ) + + # Start writer coordination task + writer_coordinator = asyncio.create_task(self._coordinate_writer_shutdown(warc_readers, warc_records_queue)) + + await job_generators + logger.info('Range jobs submitted, monitoring readers and writer') + + # Wait for all tasks to complete + readers_results = await asyncio.gather(*warc_readers) + writer_result = await warc_writer + await writer_coordinator + + readers_records = sum([result['stats']['total_records'] for result in readers_results]) + readers_mb_per_sec = self.num_readers * statistics.mean( + [result['stats']['mb_per_sec'] for result in readers_results] + ) + readers_records_per_sec = self.num_readers * statistics.mean( + [result['stats']['records_per_sec'] for result in readers_results] + ) + + logger.info(f'All WARC readers completed: {readers_records} records') + logger.info(f'Total reader throughput: {readers_mb_per_sec:.2f} MB/s; {readers_records_per_sec:.2f} rec/s') + + writer_stats = writer_result['stats'] + logger.info(f"WARC writer completed: {writer_stats['total_records']} records") + logger.info( + f"Total writer throughput: {writer_stats['mb_per_sec']:.2f} MB/s; " + f"{writer_stats['records_per_sec']:.2f} rec/s" + ) + + return writer_stats['total_records'] + + async def _coordinate_writer_shutdown(self, warc_readers: List[asyncio.Task], warc_records_queue: asyncio.Queue): + """Coordinate efficient shutdown of writers as readers complete. + + This prevents writers from waiting unnecessarily when all readers are done + and the records queue is being drained. + """ + completed_readers = 0 + + # Monitor reader completion + while completed_readers < len(warc_readers): + # Wait for any reader to complete + done, pending = await asyncio.wait( + warc_readers, + return_when=asyncio.FIRST_COMPLETED, + timeout=1.0, # Check periodically + ) + + if done: + completed_readers = len(warc_readers) - len(pending) + logger.debug(f'Readers completed: {completed_readers}/{len(warc_readers)}') + + # All readers completed - signal the writer to stop + logger.info('All readers completed, signaling writer to stop') + await warc_records_queue.put(_STOP) + + async def read_warc_records( + self, + reader_id: int, + range_jobs_queue: asyncio.Queue, + warc_records_queue: asyncio.Queue, + s3_client=None, + hf_reader=None, + ) -> dict: + """Read WARC records based on range jobs -> enqueue RangePayload. + + Args: + reader_id: Unique identifier for this reader task. + range_jobs_queue: Queue to read RangeJob objects from. + warc_records_queue: Queue to put RangePayload objects into. + s3_client: Optional S3 client for reading WARC files from S3. + hf_reader: Optional HF reader for reading WARC ranges from hf:// buckets. + + Returns: + dict: Statistics dictionary with reader_id and throughput stats. + """ + tracker = ThroughputTracker() + tracker.start() + counter = 0 + + while True: + job = await range_jobs_queue.get() + try: + if job is _STOP: + stats = tracker.get_stats() + logger.info( + 'WARC Reader %d stopping. Stats: %.1fs, %d requests, %.1f MB, %.2f MB/s, %.2f req/s', + reader_id, + stats['elapsed'], + stats['total_requests'], + stats['total_bytes'] / (1024 * 1024), + stats['mb_per_sec'], + stats['requests_per_sec'], + ) + break # Exit loop, but still execute finally block + assert isinstance(job, RangeJob) + data = await job.ranged_get_bytes( + self.max_attempts, + self.base_backoff_seconds, + s3_client=s3_client, + hf_reader=hf_reader, + ) + tracker.add(bytes_count=len(data), records_count=job.records_count) + counter += 1 + + # Log progress every N items + self.log_reader(reader_id=reader_id, counter=counter, tracker=tracker) + + await warc_records_queue.put(RangePayload(job=job, data=data)) + except Exception: + logger.exception( + 'WARC Reader %d failed on %s/%s [%d,%d]', + reader_id, + getattr(job, 'bucket', '?'), + getattr(job, 'key', '?'), + getattr(job, 'offset', -1), + getattr(job, 'length', -1), + ) + finally: + range_jobs_queue.task_done() + + return {'reader_id': reader_id, 'stats': tracker.get_stats()} + + async def write_metadata_records(self, writer, warcinfo_id: str) -> int: + """Write WARC metadata records based on paths""" + records_size = 0 + records_count = 0 + + logger.info(f'Writing {len(self.write_paths_as_metadata_records)} metadata records to WARC ... ') + + # Metadata records are written at the beginning each WARC file. + for i, resource_record_path in enumerate(self.write_paths_as_metadata_records): + logger.info(f'Writing metadata record from {resource_record_path} ...') + record = get_metadata_record_from_path( + file_path=resource_record_path, + warcinfo_id=warcinfo_id, + ) + record_data = get_bytes_from_warc_record(record, warc_version=self.warc_version, gzip=self.gzip) + await writer.write(record_data) + + # Keep track but do not rotate resource records + records_size += len(record_data) + records_count += 1 + + logger.info(f'Metadata records added: {records_count}') + + return records_size + + async def write_warc_records( + self, + warc_records_queue: asyncio.Queue, + s3_client=None, + ) -> dict: + """Write WARC records. The single writer owns the output WARC (one shard per + process) and appends ranges to it, rotating by --size when set. + + Args: + warc_records_queue: Queue to read RangePayload objects from. + s3_client: Optional S3 client for writing WARC files to S3. + + Returns: + dict: Statistics dictionary with throughput stats. + """ + # File rotation tracking; the first file is only created with the first record, so + # no WARC file is written without records + writer = None + current_file_sequence = 0 + current_file_size = 0 + + new_writer_kwargs = dict( + s3_client=s3_client, + output_path_prefix=self.prefix_path, + max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + writer_info=self.writer_info, + warc_version=self.warc_version, + writer_subprefix=self.writer_subprefix, + gzip=self.gzip, + content_type=self.content_type, + min_part_size=self.min_part_size, + ) + + tracker = ThroughputTracker() + tracker.start() + counter = 0 + + # Response records + try: + while True: + item = await warc_records_queue.get() + counter += 1 + try: + if item is _STOP: + stats = tracker.get_stats() + logger.info( + 'WARC writer stopping. Stats: %.1fs, %d items, %.1f MB written, %.2f MB/s write speed', + stats['elapsed'], + stats['total_requests'], + stats['total_bytes'] / (1024 * 1024), + stats['mb_per_sec'], + ) + should_stop = True + else: + should_stop = False + assert isinstance(item, RangePayload) + + if writer is None: + # first file, with header and metadata records + current_file_sequence = self.next_file_sequence(0) + writer, current_file_size = await self.start_file( + current_file_sequence, **new_writer_kwargs + ) + + # Check if we need to rotate files due to size limit + writer, current_file_sequence, current_file_size = await self.rotate_files( + writer=writer, + current_file_sequence=current_file_sequence, + current_file_size=current_file_size, + added_byte_size=len(item.data), + **new_writer_kwargs, + ) + + # Write actual response record + await writer.write(item.data) + current_file_size += len(item.data) + tracker.add(bytes_count=len(item.data), records_count=item.job.records_count) + + # Log progress every N items + self.log_writer(counter=counter, tracker=tracker) + + except Exception: + logger.exception('WARC writer failed on %s', getattr(item, 'job', None)) + should_stop = False + finally: + warc_records_queue.task_done() + + if should_stop: + break + finally: + if writer is not None: + await writer.close() + + return {'stats': tracker.get_stats()} + + def log_reader(self, reader_id: int, counter: int, tracker: ThroughputTracker): + """Log progress every N items.""" + if self.log_every_n > 0 and counter % self.log_every_n == 0: + stats = tracker.get_stats() + logger.info( + 'WARC Reader %d: %d items, %.1f MB, %.2f MB/s, %.2f req/s', + reader_id, + counter, + stats['total_bytes'] / (1024 * 1024), + stats['mb_per_sec'], + stats['requests_per_sec'], + ) + + def log_writer(self, counter: int, tracker: ThroughputTracker): + """Log progress every N items.""" + if self.log_every_n > 0 and counter % self.log_every_n == 0: + stats = tracker.get_stats() + logger.info( + 'WARC Writer: %d items, %.1f MB written, %.2f MB/s', + counter, + stats['total_bytes'] / (1024 * 1024), + stats['mb_per_sec'], + ) + + async def rotate_files( + self, writer, current_file_sequence: int, current_file_size: int, added_byte_size: int, **new_writer_kwargs + ): + """Check if we need to rotate files due to size limit and perform rotation if needed.""" + if self.max_file_size and current_file_size + added_byte_size > self.max_file_size: + await writer.close() + current_file_sequence = self.next_file_sequence(current_file_sequence) + writer, current_file_size = await self.start_file(current_file_sequence, **new_writer_kwargs) + logger.info(f'Rotated to new WARC file sequence {current_file_sequence} due to size limit') + + return writer, current_file_sequence, current_file_size + + async def start_file(self, sequence: int, **new_writer_kwargs): + """Start a new WARC file with its warcinfo record (and the metadata records, if any). + + Returns the writer and the number of bytes written.""" + writer, file_size, warcinfo_id = await create_new_writer_with_header(sequence=sequence, **new_writer_kwargs) + if self.write_paths_as_metadata_records: + file_size += await self.write_metadata_records(writer, warcinfo_id=warcinfo_id) + return writer, file_size diff --git a/cdx_toolkit/repackage/warc_utils.py b/cdx_toolkit/repackage/warc_utils.py new file mode 100644 index 0000000..3f7a84d --- /dev/null +++ b/cdx_toolkit/repackage/warc_utils.py @@ -0,0 +1,154 @@ +from io import BytesIO +import logging +from pathlib import Path +import fsspec +from warcio.recordloader import ArcWarcRecord +from warcio import WARCWriter + +from warcio.recordbuilder import RecordBuilder + +from typing import Dict, Optional, Tuple, Union + +import mimetypes + +from cdx_toolkit.repackage.s3_utils import is_s3_url, parse_s3_uri +from cdx_toolkit.repackage.local_writer import LocalFileWriter +from cdx_toolkit.repackage.s3_writer import S3ShardWriter + +logger = logging.getLogger(__name__) + + +def get_bytes_from_warc_record( + record, + warc_version: str = '1.0', + gzip: bool = False, +): + """Get byte representation of WARC record.""" + buffer = BytesIO() + warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) + warc_writer.write_record(record) + + return buffer.getvalue() + + +def get_metadata_record_from_path( + file_path: Union[str, Path], + warcinfo_id: str, +) -> ArcWarcRecord: + """Build WARC metadata record for file path. + + The `Content-Type` header is guessed. + """ + # Cast to string + file_path = str(file_path) + + with fsspec.open(file_path, 'rb') as f: + file_bytes = BytesIO(f.read()) + + warc_content_type = mimetypes.guess_type(file_path)[0] + warc_headers_dict = {} + + # Set WARC-Warcinfo-ID + warc_headers_dict['WARC-Warcinfo-ID'] = warcinfo_id + + rb = RecordBuilder() + record = rb.create_warc_record( + uri=None, + record_type='metadata', + payload=file_bytes, + http_headers=None, + warc_content_type=warc_content_type, + warc_headers_dict=warc_headers_dict, + ) + + return record + + +def generate_warc_filename( + dest_prefix: str, + sequence: int, + writer_subprefix: Optional[str] = None, + gzip: bool = False, +) -> str: + """Generate a WARC file name from a prefix (can be a full path), an optional subprefix + (used to distinguish per-process shards), and a sequence index (file-rotation counter).""" + file_name = dest_prefix + '-' + if writer_subprefix is not None: + file_name += writer_subprefix + '-' + file_name += f'{sequence:03d}.warc' # TODO default warc command uses ".extracted.warc" + if gzip: + file_name += '.gz' + + return file_name + + +async def create_new_writer_with_header( + sequence: int, + output_path_prefix: str, + max_attempts: int, + base_backoff_seconds: float, + min_part_size: int, + writer_info: Dict, + warc_version: str = '1.0', + writer_subprefix: Optional[str] = None, + gzip: bool = False, + content_type: Optional[str] = None, + s3_client=None, +) -> Tuple[Union[S3ShardWriter, LocalFileWriter], int, str]: + """Create a new WARC writer (local or S3) and write the warcinfo record. + + Returns the writer, the size of the warcinfo record and its WARC-Record-ID.""" + if is_s3_url(output_path_prefix): + dest_bucket, dest_prefix = parse_s3_uri(output_path_prefix) + + warc_file_path = generate_warc_filename( + dest_prefix=dest_prefix, + sequence=sequence, + writer_subprefix=writer_subprefix, + gzip=gzip, + ) + + new_writer = S3ShardWriter( + s3_client, + warc_file_path, + dest_bucket, + content_type, + min_part_size, + max_attempts, + base_backoff_seconds, + ) + + else: + # local file system + warc_file_path = generate_warc_filename( + dest_prefix=output_path_prefix, + sequence=sequence, + writer_subprefix=writer_subprefix, + gzip=gzip, + ) + + new_writer = LocalFileWriter( + file_path=warc_file_path, + ) + + logger.debug('Initialzing new WARC writer for %s', warc_file_path) + + # Initialize writer + await new_writer.start() + + # Write WARC header; the `filename` field names this file + warc_file_name = warc_file_path.split('/')[-1] + buffer = BytesIO() + warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) + warcinfo = warc_writer.create_warcinfo_record( + filename=warc_file_name, + info=writer_info, + ) + warc_writer.write_record(warcinfo) + header_data = buffer.getvalue() + await new_writer.write(header_data) + + # WARC-Warcinfo-ID indicates the WARC-Record-ID of the associated ‘warcinfo’ record + warcinfo_id = warcinfo.rec_headers.get('WARC-Record-ID') + + return new_writer, len(header_data), warcinfo_id diff --git a/cdx_toolkit/timeutils.py b/cdx_toolkit/timeutils.py index ed4b5b9..3540710 100644 --- a/cdx_toolkit/timeutils.py +++ b/cdx_toolkit/timeutils.py @@ -1,3 +1,4 @@ +import calendar import datetime from email.utils import parsedate import logging @@ -12,9 +13,6 @@ TIMESTAMP_LOW = '19780101000000' TIMESTAMP_HIGH = '29991231235959' -# if you ask for Feb we'll pad it to the 28th even if it's a leap year -days_in_month = (0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) - def pad_timestamp(ts): ''' @@ -28,11 +26,16 @@ def pad_timestamp_up(ts): ''' >>> pad_timestamp_up('199802') '19980228235959' + >>> pad_timestamp_up('200002') + '20000229235959' + >>> pad_timestamp_up('20260410') + '20260410235959' ''' - ts = ts + TIMESTAMP_HIGH[len(ts):] - month = ts[4:6] - ts = ts[:6] + str(days_in_month[int(month)]) + ts[8:] - return ts + padded = ts + TIMESTAMP_HIGH[len(ts):] + if len(ts) < 8: # no day given: the last day of the month + last_day = calendar.monthrange(int(padded[:4]), int(padded[4:6]))[1] + padded = padded[:6] + f'{last_day:02d}' + padded[8:] + return padded def timestamp_to_time(ts): diff --git a/cdx_toolkit/utils.py b/cdx_toolkit/utils.py index aaf460c..9f6d56d 100644 --- a/cdx_toolkit/utils.py +++ b/cdx_toolkit/utils.py @@ -1,50 +1,5 @@ import cdx_toolkit -from cdx_toolkit.commoncrawl import normalize_crawl - -from argparse import Namespace -import logging - -LOGGER = logging.getLogger(__name__) def get_version(): return cdx_toolkit.__version__ - - -def setup_cdx_fetcher_and_kwargs(cmd: Namespace): - kwargs = {} - kwargs['source'] = 'cc' if cmd.crawl else cmd.cc or cmd.ia or cmd.source or None - if kwargs['source'] is None: - raise ValueError('must specify --cc, --ia, or a --source') - if cmd.wb: - kwargs['wb'] = cmd.wb - if cmd.cc_mirror: - kwargs['cc_mirror'] = cmd.cc_mirror - if cmd.crawl: - kwargs['crawl'] = normalize_crawl([cmd.crawl]) # currently a string, not a list - if getattr(cmd, 'warc_download_prefix', None) is not None: - kwargs['warc_download_prefix'] = cmd.warc_download_prefix - - cdx = cdx_toolkit.CDXFetcher(**kwargs) - - kwargs = {} - if cmd.limit: - kwargs['limit'] = cmd.limit - if 'from' in vars(cmd) and vars(cmd)['from']: # python, uh, from is a reserved word - kwargs['from_ts'] = vars(cmd)['from'] - if cmd.to: - kwargs['to'] = cmd.to - if cmd.closest: - if not cmd.get: # pragma: no cover - LOGGER.info('note: --closest works best with --get') - kwargs['closest'] = cmd.closest - if cmd.filter: - kwargs['filter'] = cmd.filter - - if cmd.cmd == 'warc' and cmd.size: - kwargs['size'] = cmd.size - - if cmd.cmd == 'size' and cmd.details: - kwargs['details'] = cmd.details - - return cdx, kwargs diff --git a/requirements.txt b/requirements.txt index dba64ea..2901643 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,10 +3,28 @@ requests>=2.25.1 warcio==1.7.4 +fsspec +aiofiles +url-is-in>=0.1.1 # optional S3 dependencies (install via cdx_toolkit[s3]) fsspec[s3] botocore +aioboto3 + +# optional Athena dependencies (install via cdx_toolkit[athena]) +boto3 + +# optional DuckDB dependencies (install via cdx_toolkit[duckdb]) +duckdb +pytz + +# optional Hugging Face dependencies (install via cdx_toolkit[hf]) +huggingface_hub>=0.30 +aiohttp + +# optional Parquet dependencies (install via cdx_toolkit[parquet]) +pyarrow # used by Makefile pytest>=6.2.4 diff --git a/scripts/cdx_iter b/scripts/cdx_iter deleted file mode 100644 index 99445c0..0000000 --- a/scripts/cdx_iter +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python - -import logging -import argparse -import json -import csv -import sys -import os.path - -import cdx_toolkit -import cdx_toolkit.warc - -print('This command is deprecated in favor of cdxt', file=sys.stderr) - -ARGS = argparse.ArgumentParser(description='cdx_toolkit iterator command line tool') -ARGS.add_argument('--cc', action='store_const', const='cc', help='direct the query to the Common Crawl CDX/WARCs') -ARGS.add_argument('--ia', action='store_const', const='ia', help='direct the query to the Internet Archive CDX/wayback') -ARGS.add_argument('--source', action='store', help='direct the query to this CDX server') -ARGS.add_argument('--wb', action='store', help='direct replays for content to this wayback') -ARGS.add_argument('--limit', type=int, action='store') -ARGS.add_argument('--cc-mirror', action='store', help='use this Common Crawl index mirror') -ARGS.add_argument('--cc-sort', action='store', help='default mixed, alternatively: ascending') -ARGS.add_argument('--from', action='store') -ARGS.add_argument('--to', action='store') -ARGS.add_argument('--closest', action='store') # this is a get() thing, not an iter thing -ARGS.add_argument('--filter', action='store', help='see CDX API documentation for usage') -ARGS.add_argument('--all-fields', action='store_true') -ARGS.add_argument('--fields', action='store', default='url,status,timestamp', help='try --all-fields if you need the list') -ARGS.add_argument('--jsonl', action='store_true') -ARGS.add_argument('--csv', action='store_true') -ARGS.add_argument('--warc', action='store_true', help='create a warc of the output contents') -ARGS.add_argument('--warc-prefix', default='TEST', help='prefix for the warc filename') -ARGS.add_argument('--warc-subprefix', type=str, default=None, help='subprefix for the warc filename, default None') -ARGS.add_argument('--warc-size', type=int, default=1000000000, help='target for the warc filesize in bytes') -ARGS.add_argument('--warc-creator', action='store', help='creator of the warc: person, organization, service') -ARGS.add_argument('--warc-operator', action='store', help='a person, if the creator is an organization') -ARGS.add_argument('--warc-url-fgrep', action='store', help='this pattern must be present to warc an url') -ARGS.add_argument('--warc-url-fgrepv', action='store', help='this pattern must not be present to warc an url, e.g. /robots.txt') -ARGS.add_argument('--get', action='store_true', help='use a single get instead of a paged iteration') -ARGS.add_argument('--verbose', '-v', action='count', help='set logging level to INFO (-v) or DEBUG (-vv)') -ARGS.add_argument('url') - -cmdline = ' '.join(sys.argv[1:]) -args = ARGS.parse_args() - -loglevel = os.getenv('LOGLEVEL') or 'WARNING' -if args.verbose: - if args.verbose > 0: - loglevel = 'INFO' - if args.verbose > 1: - loglevel = 'DEBUG' -logging.basicConfig(level=loglevel) -LOGGER = logging.getLogger(__name__) - -if not args.url: - raise ValueError('must specify an url to iterate, example: commoncrawl.org/*') - -kwargs = {} -kwargs['source'] = args.cc or args.ia or args.source or None -if kwargs['source'] is None: - raise ValueError('must specify --cc, --ia, or a --source') -if args.wb: - kwargs['wb'] = args.wb -if args.cc_mirror: - kwargs['cc_mirror'] = args.cc_mirror - -cdx = cdx_toolkit.CDXFetcher(**kwargs) - -fields = set(args.fields.split(',')) - -kwargs = {} -if args.limit: - kwargs['limit'] = args.limit -if 'from' in vars(args) and vars(args)['from']: # python, uh, from is a reserved word - kwargs['from_ts'] = vars(args)['from'] -if args.to: - kwargs['to'] = args.to -if args.closest: - if not args.get: - LOGGER.info('note: --closest works best with --get') - kwargs['closest'] = args.closest -if args.filter: - kwargs['filter'] = args.filter - -if args.csv: - writer = csv.DictWriter(sys.stdout, fieldnames=sorted(list(fields))) - writer.writeheader() - - -def winnow_fields(obj): - if args.all_fields: - printme = obj - else: - printme = dict([(k, obj[k]) for k in fields if k in obj]) - return printme - - -def print_line(printme): - if args.jsonl: - print(json.dumps(printme, sort_keys=True)) - elif args.csv: - writer.writerow(printme) - else: - print(', '.join([' '.join((k, printme[k])) for k in sorted(printme.keys())])) - - -if args.get: - objs = cdx.get(args.url, **kwargs) - for obj in objs: - printme = winnow_fields(obj) - print_line(printme) -elif args.warc: - info = { - 'software': 'pypi_cdx_toolkit/'+cdx_toolkit.__version__, - 'isPartOf': args.warc_prefix, - 'description': 'warc extraction generated with: cdx_toolkit '+cmdline, - 'format': 'WARC file version 1.0', # todo: if we directly read a warc, have this match the warc - } - if args.warc_creator: - info['creator'] = args.warc_creator - if args.warc_operator: - info['operator'] = args.warc_operator - - writer_kwargs = {} - writer_kwargs['size'] = args.warc_size - - writer = cdx_toolkit.warc.get_writer(args.warc_prefix, args.warc_subprefix, info, **writer_kwargs) - - for obj in cdx.iter(args.url, **kwargs): - url = obj['url'] - if args.warc_url_fgrep and args.warc_url_fgrep not in url: - LOGGER.debug('not warcing due to fgrep: %s', url) - continue - if args.warc_url_fgrepv and args.warc_url_fgrepv in url: - LOGGER.debug('not warcing due to fgrepv: %s', url) - continue - timestamp = obj['timestamp'] - try: - record = obj.fetch_warc_record() - except RuntimeError: # pragma: no cover - LOGGER.warning('skipping capture for RuntimeError 404: %s %s', url, timestamp) - continue - if obj.is_revisit(): - LOGGER.warning('revisit record being resolved for url %s %s', url, timestamp) - writer.write_record(record) - - writer.close() -else: - for obj in cdx.iter(args.url, **kwargs): - printme = winnow_fields(obj) - print_line(printme) diff --git a/scripts/cdx_size b/scripts/cdx_size deleted file mode 100644 index 1135f80..0000000 --- a/scripts/cdx_size +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python - -import os -import logging -import argparse -import sys - -import cdx_toolkit - -print('This command is deprecated in favor of cdxt', file=sys.stderr) - -loglevel = os.getenv('LOGLEVEL') or 'WARNING' -logging.basicConfig(level=loglevel) - -ARGS = argparse.ArgumentParser(description='cdx_toolkit size command line tool') -ARGS.add_argument('--cc', action='store_true') -ARGS.add_argument('--ia', action='store_true') -ARGS.add_argument('url') - -args = ARGS.parse_args() - -if not (args.cc or args.ia): - raise ValueError('must specify one of --cc or --ia') - -if args.cc: - cdx = cdx_toolkit.CDXFetcher(source='cc') - print('cc size estimate', cdx.get_size_estimate(args.url)) - -if args.ia: - cdx = cdx_toolkit.CDXFetcher(source='ia') - print('ia size estimate', cdx.get_size_estimate(args.url)) diff --git a/setup.py b/setup.py index b707eb6..ce6eb86 100755 --- a/setup.py +++ b/setup.py @@ -8,10 +8,19 @@ packages = find_packages(include=['cdx_toolkit*']) # remember: keep requires synchronized with requirements.txt -requires = ['requests', 'warcio'] +requires = ['requests', 'warcio', 'fsspec', 'aiofiles', 'url-is-in>=0.1.1'] test_requirements = ['pytest', 'pytest-cov', 'flake8', 'responses'] -optional_s3_requirements = ['fsspec[s3]', 'botocore'] +# S3 reads/writes (index files, WARCs) +optional_s3_requirements = ['fsspec[s3]', 'botocore', 'aioboto3'] +# `cdxt index athena://...` +optional_athena_requirements = ['boto3'] +# `cdxt index cc-url-index-s3` and other parquet indexes +optional_duckdb_requirements = ['duckdb', 'pytz'] # pytz: DuckDB returns timestamps with time zone +# Hugging Face Storage Buckets (hf://): `cdxt index cc-url-index-hf`, `cdxt repackage` reads and writes +optional_hf_requirements = ['huggingface_hub>=0.30', 'aiohttp'] +# Parquet output of `cdxt index` (and Parquet input of `cdxt repackage`) +optional_parquet_requirements = ['pyarrow'] package_requirements = ['twine', 'setuptools', 'setuptools-scm'] @@ -19,14 +28,20 @@ extras_require = { 's3': optional_s3_requirements, + 'athena': optional_athena_requirements, + 'duckdb': optional_duckdb_requirements, + 'hf': optional_hf_requirements, + 'parquet': optional_parquet_requirements, 'test': test_requirements, # setup no longer tests, so make them an extra 'package': package_requirements, 'dev': package_requirements, - 'all': test_requirements + package_requirements + dev_requirements + optional_s3_requirements, + 'all': ( + test_requirements + package_requirements + dev_requirements + + optional_s3_requirements + optional_athena_requirements + + optional_duckdb_requirements + optional_hf_requirements + optional_parquet_requirements + ), } -scripts = ['scripts/cdx_size', 'scripts/cdx_iter'] - this_directory = path.abspath(path.dirname(__file__)) with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: description = f.read() @@ -48,9 +63,7 @@ entry_points=''' [console_scripts] cdxt = cdx_toolkit.cli:main - #ccathena = cdx_toolkit.cli:main_athena ''', - scripts=scripts, license='Apache 2.0', classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/conftest.py b/tests/conftest.py index 9f54f82..ce0ce91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,18 @@ import json import os +import uuid from pathlib import Path import pytest from cdx_toolkit.settings import CACHE_DIR +try: + import boto3 + _HAS_BOTO3 = True +except ImportError: # pragma: no cover - exercised in minimal installs + boto3 = None + _HAS_BOTO3 = False + try: import botocore.session from botocore.config import Config @@ -22,6 +30,12 @@ except ImportError: # pragma: no cover - exercised in minimal installs _HAS_FSSPEC = False +try: + import duckdb # noqa: F401 + _HAS_DUCKDB = True +except ImportError: # pragma: no cover - exercised in minimal installs + _HAS_DUCKDB = False + import functools from typing import Dict, Optional import requests @@ -31,13 +45,17 @@ from unittest.mock import patch - +TEST_DATA_PATH = Path(__file__).parent / 'data' TEST_DATA_BASE_PATH = Path(__file__).parent / 'data' TEST_S3_BUCKET = os.environ.get('CDXT_TEST_S3_BUCKET', 'commoncrawl-ci-temp') +TEST_ATHENA_S3_LOCATION = 's3://commoncrawl-ci-temp/athena-results/' +TEST_ATHENA_DATABASE = 'ccindex' +DISABLE_ATHENA_TESTS = bool(os.environ.get('CDXT_DISABLE_ATHENA_TESTS', False)) DISABLE_S3_TESTS = bool(os.environ.get('CDXT_DISABLE_S3_TESTS', False)) -# Cache for AWS access check to avoid repeated network calls +# Cache for AWS S3/Athena access check to avoid repeated network calls _aws_s3_access_cache = None +_aws_athena_access_cache = None @pytest.fixture(scope='session', autouse=True) @@ -94,6 +112,96 @@ def requires_aws_s3(func): ) +def check_aws_athena_query_execution_access(): + """Check if AWS Athena StartQueryExecution permission is available.""" + try: + # Use IAM simulation instead of actual query execution + iam_client = boto3.client('iam') + response = iam_client.simulate_principal_policy( + PolicySourceArn=f'arn:aws:sts::{boto3.client("sts").get_caller_identity()["Account"]}:assumed-role/your-role/session', # noqa: E501 + ActionNames=['athena:StartQueryExecution'], + ResourceArns=['*'], + ) + + # Check if access is allowed + return response['EvaluationResults'][0]['EvalDecision'] == 'allowed' + + except (ClientError, NoCredentialsError): + return False + + +def check_aws_athena_access(): + """Check if AWS Athena access is available.""" + global _aws_athena_access_cache + + if not _HAS_BOTO3: + return False + + if _aws_athena_access_cache is not None: + return _aws_athena_access_cache + + try: + client = boto3.client('athena') + + # Try list databasets + client.list_databases(CatalogName='AwsDataCatalog') + + # Try query access + _aws_athena_access_cache = check_aws_athena_query_execution_access() + except (NoCredentialsError, ClientError): + _aws_athena_access_cache = False + + return _aws_athena_access_cache + + +def requires_aws_athena(func): + """Pytest decorator that skips test if AWS Athena access is not available.""" + if not _HAS_BOTO3: + return pytest.mark.skipif( + True, reason='S3 dependencies are not installed; install cdx_toolkit[s3] to enable Athena tests.' + )(func) + + return pytest.mark.skipif(DISABLE_ATHENA_TESTS, reason='AWS Athena access is disabled via environment variable.')( + pytest.mark.skipif( + not check_aws_athena_access(), reason='AWS Athena access not available (no credentials or permissions)' + )(func) + ) + + +def requires_duckdb(func): + """Pytest decorator that skips a test if the optional duckdb dependency is missing.""" + return pytest.mark.skipif( + not _HAS_DUCKDB, reason='duckdb is not installed; install cdx_toolkit[duckdb] to enable DuckDB tests.' + )(func) + + +@pytest.fixture +def s3_tmpdir(): + """S3 equivalent of tmpdir - provides a temporary S3 path and handles cleanup.""" + bucket_name = TEST_S3_BUCKET + + # Generate unique prefix using UUID to avoid collisions + temp_prefix = f'cdx_toolkit/ci/tmpdirs/{uuid.uuid4().hex}' + + # Yield the S3 path + yield f's3://{bucket_name}/{temp_prefix}' + + try: + # Cleanup: delete all objects with this prefix + s3_client = boto3.client('s3') + + # List all objects with the temp prefix + response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=temp_prefix) + + if 'Contents' in response: + # Delete all objects + objects_to_delete = [{'Key': obj['Key']} for obj in response['Contents']] + s3_client.delete_objects(Bucket=bucket_name, Delete={'Objects': objects_to_delete}) + except (NoCredentialsError, ClientError, ConnectionError, EndpointConnectionError): + # Ignore cleanup errors - test objects will eventually expire + pass + + def flexible_param_matcher(expected_params): """Custom matcher that ignores dynamic 'from' parameter timestamps and casts all values to strings""" @@ -195,28 +303,39 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None ) -def conditional_mock_responses(func): +def conditional_mock_responses(func=None, *, auto_mock_data: bool = True): """Conditionally applies @responses.activate and auto-loads mock data based on DISABLE_MOCK_RESPONSES env var. The mock data is automatically loaded from JSONL file from the tests/data directory and dependinng on the test module and test function. + + Args: + auto_mock_data: If True, auto-loads test-specific mock data. If False, only loads CC endpoints. """ - # If the flag DISABLE_MOCK_RESPONSES is not detected, response mocking remains enabled - if not os.environ.get('DISABLE_MOCK_RESPONSES'): - # Add responses.activate - func = add_mock_responses(func) + def decorator(f): + # If the flag DISABLE_MOCK_RESPONSES is not detected, response mocking remains enabled + if not os.environ.get('DISABLE_MOCK_RESPONSES'): + # Add responses.activate + f = add_mock_responses(f, auto_mock_data=auto_mock_data) + + if os.environ.get('SAVE_MOCK_RESPONSES'): + # Mock data is saved by capturing output from requests.get + @functools.wraps(f) + def wrapper(*args, **kwargs): + with patch('requests.get', side_effect=_custom_behavior_with_original(requests.get)): + return f(*args, **kwargs) - if os.environ.get('SAVE_MOCK_RESPONSES'): - # Mock data is saved by capturing output from requests.get - @functools.wraps(func) - def wrapper(*args, **kwargs): - with patch('requests.get', side_effect=_custom_behavior_with_original(requests.get)): - return func(*args, **kwargs) + return wrapper - return wrapper + return f - return func + if func is None: + # Called with arguments: @conditional_mock_responses(auto_mock_data=False) + return decorator + else: + # Called without arguments: @conditional_mock_responses + return decorator(func) def save_response_as_mock_data(test_info: str, request_url: str, request_params: Dict, resp, output_base_dir: str): @@ -283,14 +402,15 @@ def custom_behavior(*args, **kwargs): return custom_behavior -def add_mock_responses(func): +def add_mock_responses(func, auto_mock_data: bool = True): @functools.wraps(func) def wrapper(*args, **kwargs): # Load mock data for index calls (same for many test functions) mock_response_from_jsonl('test_get_cc_endpoints', 'test_cc') # Auto-load mock data based on function name - mock_response_from_jsonl(func.__name__, func.__module__.split('.')[-1]) + if auto_mock_data: + mock_response_from_jsonl(func.__name__, func.__module__.split('.')[-1]) return func(*args, **kwargs) return responses.activate(wrapper) diff --git a/tests/data/mock_responses/test_cli/test_iter_alias.jsonl b/tests/data/mock_responses/test_cli/test_iter_alias.jsonl new file mode 100644 index 0000000..10fa19f --- /dev/null +++ b/tests/data/mock_responses/test_cli/test_iter_alias.jsonl @@ -0,0 +1 @@ +{"method": "GET", "url": "https://index.commoncrawl.org/CC-MAIN-2025-33-index", "request_params": {"limit": 10, "url": "commoncrawl.org/*", "output": "json", "page": 0, "from": "20240915202343"}, "response_status_code": 200, "response_text": "{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804103329\", \"url\": \"https://www.commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"FAPYCDZSWJIQ2YPQAYDRLAC25YMUNVTD\", \"length\": \"7063\", \"offset\": \"667474763\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279781.29/warc/CC-MAIN-20250804083340-20250804113340-00340.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804145408\", \"url\": \"http://commoncrawl.org\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"570\", \"offset\": \"71720\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279808.15/robotstxt/CC-MAIN-20250804145214-20250804175214-00905.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804145409\", \"url\": \"http://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"571\", \"offset\": \"138028\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279808.15/crawldiagnostics/CC-MAIN-20250804145214-20250804175214-00494.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804145409\", \"url\": \"https://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"GWWHOAM46QU5UTPSJAUTVADAM56EUVPS\", \"length\": \"7055\", \"offset\": \"128243158\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279808.15/warc/CC-MAIN-20250804145214-20250804175214-00389.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804180633\", \"url\": \"http://commoncrawl.org\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"568\", \"offset\": \"69067\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279827.21/robotstxt/CC-MAIN-20250804180035-20250804210035-00905.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804180633\", \"url\": \"http://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"568\", \"offset\": \"213023\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279827.21/crawldiagnostics/CC-MAIN-20250804180035-20250804210035-00494.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804180633\", \"url\": \"https://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"UQZZ7W2BXQXNZGXL323YV3RKVIVVYTLL\", \"length\": \"6977\", \"offset\": \"144661303\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279827.21/warc/CC-MAIN-20250804180035-20250804210035-00389.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250805042626\", \"url\": \"http://commoncrawl.org\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"570\", \"offset\": \"60915\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279847.20/robotstxt/CC-MAIN-20250805032822-20250805062822-00905.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250805042627\", \"url\": \"http://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"571\", \"offset\": \"173482\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279847.20/crawldiagnostics/CC-MAIN-20250805032822-20250805062822-00494.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250805042627\", \"url\": \"https://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"UQZZ7W2BXQXNZGXL323YV3RKVIVVYTLL\", \"length\": \"6986\", \"offset\": \"134133724\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279847.20/warc/CC-MAIN-20250805032822-20250805062822-00389.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n", "response_headers": {"Server": "nginx/1.29.1", "Date": "Mon, 15 Sep 2025 20:23:44 GMT", "Content-Type": "text/x-ndjson", "Transfer-Encoding": "chunked", "Connection": "keep-alive", "Content-Encoding": "gzip"}} diff --git a/tests/data/mock_responses/test_cli/test_multi_misc_not_ia_slow_5.jsonl b/tests/data/mock_responses/test_cli/test_multi_misc_not_ia_slow_5.jsonl deleted file mode 100644 index 7bd7eab..0000000 --- a/tests/data/mock_responses/test_cli/test_multi_misc_not_ia_slow_5.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"method": "GET", "url": "https://index.commoncrawl.org/CC-MAIN-2025-33-index", "request_params": {"url": "commoncrawl.org/*", "showNumPages": "true", "limit": 10, "from": "20240915203928"}, "response_status_code": 200, "response_text": "{\"pages\": 1, \"pageSize\": 5, \"blocks\": 1}", "response_headers": {"Server": "nginx/1.29.1", "Date": "Mon, 15 Sep 2025 20:39:30 GMT", "Content-Type": "text/x-cdxj", "Transfer-Encoding": "chunked", "Connection": "keep-alive"}} diff --git a/tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.gz b/tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.gz new file mode 100644 index 0000000000000000000000000000000000000000..47941edb9c3bb5f953bedd593380e8ef00bc3503 GIT binary patch literal 63684 zcmV)PK()UgiwFqz+@)v&|6^o$Eif=KI5z;?y=#vpx0NmUeSSs59}En}mV@NpWbgvU zAl~nH1mp3gA=_nJfGm5g(%imxF#mlRsgza4$dkHtP6SW)xPi7$sg#kKE4aBIYp?a^ zfBxog-+cf5H-G=`PappL&#%Axk;;hR5v`_11!{`JHE^}~n%{deE!fAAk){o}9y z_&>k-)9)FPzy9|B{`U8O_~F}cKYaY=yMO=FH{bu`+pqt}zfp&RBM2wR2;#r}Kfn6n zpWpuKzx~^<{`&FbKfeEOzx&;%U-K*DfnWXkpPxMCtuO!UA0Pdz-~8LJ{`Sq^KI~uk zG4`nUV9>$lAL-S;2<^0yD)ef<7+L{LX?h6Du4ubBAVfB*WQfBfvPzx;NAQUX7J z6fYkghcWTh{x5v>m;bpx{EvVA`k$}g)HKGwbzi>oufO}t4`1t_b^lZSd;it{`@jA9 zOQFXX?7#lu<6r;&&wu~^!-t6 zwJ)I(@;S2&;J%_wL*9z@$^9U$W8kNoAM<4}a_(WcG~u+c$> zqc|6A>RgL;xfN~MA#3}%?($a6S<%)jlzCR9eh_V+V@(_(Tmx*z(RJO&z~=gBQ~D;@ z=zt>`{QOb8d~_Tn!W!5wfB4OJA3r67R zoL+o;$Es9SA4G|Y>D0z)7*q_`b=&HwYJnSg3qp+p&h-g*$OCA*`g%tCmFvERw*IXO zM8JQ2q|VDnXTQ9^6z$*WpW`0H#f^}z-$O!~eDsVTAQ#s{-qum?GZ=X~Oq*b;P8p|| zrt`SAx&C7_s8eQOP8`VrfIWD|^{?-%x36&hPxMpqUmx|4uz%E`e|c2@&9`5FyaRM{ z19Ph54`uGzn3HQEhhU>Ibe$Tyd`%9oG(K_?c%I(fG~)2LJKJes_{S)N@D{pi)3Kj&(wYHw4c9|vha zEz7*-Jm7s77y%y50M9jclJFeWx40&4>lh>pH7b@TCzMQZw z@^JEF8Z1$co>8HpkY*0YrNbOT?5FKIgZg67^*4AA=6WBKl`FM{OUc$L2+9dq7p0mG_d226M4J&10-XoZmi#M9-bQ;bg#UHass0gOKI;GCj-~j{ zZdd7q0eigfdaTkf#@(;Iw)IP|Y*n#hx&T*2Q;3q~Lr9Yb%@iKYzKl-cUvS~}+i-W> z>vv%OW6$-EfB*K|&nq@pCpeGhaX)?4AECau-5Pd_s+ytm>%eQ57}7OYRl0Ox>2+1L z`bK+QU!$KRji7(;60MhN2*lFyM(1{|2Es)9@$?NZZoxHeh%l8?zpx=*_~6CiP~;rC z2?CmJ!pPXAIF#o^=u*u^`#C$|XbW_4bEpE2{^?H#{o#pN+aCj>9RO+b>fiFqM-91Tb|HVWPJ5Y~JC3-b zPP-cWxX8N|2ex8V^Lom0NIO1Fo9J)}!g(<2H0qGU*qlq0@(E+}JI?zZ!1G&UbLqI~ zn8o?SXPsZ1$HnPd)hxk9jJ_HmOzT<1surynL^x#z-Z~5$@tt$-x}4ZQ>XhCU-GB4# z$NzZ;>f&atreWVlix0wnEY&VP=a6_=9tU%l4@Do$Y*s9-mOQ68jw)O2540{-+N=+yS~PZJoSnDJt9$H8*kHJ>bI_$3PvS@g-ZWHe zrl6(8mKzoNYxMpkZI`YqD6=2Z_5ZhlwtFL_13}7wpMAQedvOb{-u$s0w~Z>whOqU< zwFs89>Qo(j`Hu~^PeM9%4$W3B_EywmG&omR-(L}UGo-^y(?kCJQM`QA$6H!B-t5=k z{`<%O@sB_K{g1!@`t#w+umAS-H{bpK*Wdl&pa1^tH-Gu=!}q_J&aeNWANG6tNdzGJ ze>~cf{cP;QUY}t+_{VSN=kN8}wqyzR873Fs2s;iU{NvNl`38e|1g{Rk@xfS zYXckzbLkTjKXm?czBK&Pb`NFZikF5z^`%LN7^uBCiZMiTuuIA9*UDP$udm@(C}^XXV`Cy%Bf}HahL0r_cM|3OM@VcHbSfP=D)ioVJKX4f3Rz z)_$Hcm31MV%0o9%-wlVs0dt6aPCrrnr+s_yleY)R~V=}ryI@QrwQ()e;r^hS>s|Clg3IFW%ZKL z3B%4SRY>P~7JL+aFou)9g3SKpkLAm!@Vi3NS9+6O;y;rX7^1Jxo4lcY$c*A~61b6z zTnzRsR66);MXR=B%?!Gf;R9VVsM40tV{aaI35OXo6B=*wgpLJsr8Ttw@q_*=-~aW) zXFUt9zi;~R{Nb}NZ3T2)z~)IIaO>2RG##(Es$pIxR*y|_y{_g7=kB2%aRhSv^CsuZ z!}~pm%Ug|TU7za2=;W@&<$Jlxi!)d$jf!O5ingXKnR}6|MHTsknL+$^`LS!^B6VGd z+b4qRV%XAsPH&Ez^yNZzUUBxgiCw(UmBuY)TQvK9moA{*uVeb4+beze_u>T<40!@pL}6Ec#p*p7q|^xO4kOKBmf`t zw(e$8@h~y?1e~L}xVaNyC9{tzc7uFthJ?cucOBN= zl*gqS2X&;132rj|by$}A#}`jH-(UlA_7>xt5@*lWA1}{7JK7hs`kEQjI+6oPtCzH*;ltTG+;kxM~OBL zIe(W8%9-iw?Qt=X?=V8Sb25uWKknjEtyqg;Dp(^_srnX09{N`-OQrNfL;4F#u6 zh-WZDzkJK%E;cB`8#(*FK#VRveAEv~`^6++zpMuq_Bm^sB(KY^kC@8p9HzBfrgUZp zo|qZm$sXVyWs=;rnayqKt*~259y@&q^*wWT>V7rTzbKQo-(oLPUbl>xdQHAyuEs=^ zIDx6azy%S9*zO2dKDeuSAAQIg==^3QZWm4y3{P*wU3{Z^aV2uD)`a3%`lDL9GWVlK zgcDn)HXh6*9?gX=U8VAe*SYS}U_fdM^jox0ooq0P?Cg`yFXn}&(9*ufrQu`a_vtjZ z#fV#wYHJfCQG6CPp-o?eexO!1ZSqR4S z*qhQNDO%)P&BtL1x?%lscKT`d&WWj#s8cOK`|+KjjajPuWxN{zgSAC>oi*Z|&%1Ho zpl#`=+a=zFovMbk?o)rr=g^CzK-5v#!03xSi}U)h1ZsBnq$CVn_JmQq088uM%)OZ& zaDu==Uhr~U3wt#W#lS>Z2YggrE2>J-H1^k|%93eGrvu;WK)BJz!le?NCk*7lR$3@; zZu#r95Kr&oUtICyT=+9KO{*er>Y{|FN(}|{g=*`D#h5bo?h{m7taW+h z+dL~*r-VSsxs>m#J3M2&VupR&2jN<=Bxh|w%PC`dR8}(>cuQvP0y4gLSDAgZ!3vpM z-U{mave!ijxqxYVG2m@FkHHo;S=x_HA1x8mqJgwiIJRBJkCPjaM-nY{UkeqxlN$%X z&IyMb2b?-Ot>EVpDK9SgiY*;^0Y#vOsGs~~2s*m2_y>ZAG^N{N$LA=uh{$w)<=SV> zQnWz12j@Ncj(r^i5zepp;A#zJHuKi=pkUtWMNKx*pj0AFbzH|hD$>fR_#9^74yf;O zf=}q(I~KTB>e22ACNZ+KK*dJ1t0fm*c4@xC}^Uk>9V{k|MFfOji4cyO)ju2F<8 zRBQwwxGfvX0M7RH(9gSAWTN#dppn;1P1%h35_bKz6+#4c16{tFOFk&&0q|Wk^ik!Z zyHrD^jY{M_?Qe+^E{Fsdb42;%djX1cQGT^mahtS#kIgVQjKzVzJPvFA+{`lc53h?) z_6&#Z66kK&t?(PiZyiLtyivon|CW6rhGjRN$@}l zrmidGj~Zxpa|3Q`kq-cGlEyfdkNY~9fgLVp3bvx(C4u{@&Wmp5dB}$Vi(X}_UPnG3 zeE6Khx`*THobV}{Q#Pn1`r5j6lhfT=;$(SP{kV%qgZ(rQ{m`zBkG?7eZ^Lxo@ma#E zo?Fix4N7Soa{Zb!_t9DKUE&Llw1M{RP64N`b2cYbKks6V|9lwJF-_N~OqXd2x3R11 zwWg|JydIgc&Lv+v7oT+sUaaWY6PRL>TZ-woK;Mfv#?Mxbyd3j%D}odY%9}jiV3|Zr zsh6&ptEy_MemqQrO|f@L7(S>Pc^^xRSaM3F%g*8wF*+vcAY50 zyuy}aTo!3FihN^{Nd)GCz;PtZ!3ANzMAMm*9lK4qDfYg1f+>+FAN8xn-rG4TQnq&O zhAio$sH)PY3C5usR<0;DMm!Q-Qy|ve#m=8V17=`BAQt#Lw>e>Ow&u5f-of_GeUEiC^s>&p|EsDa`Z^IJ|dl;e(;NN#@0&vm21o(@& zUP~6VQ`~t;m}Xuc&9G!;;T6fo7#GH!-=^SYo(6R#KXHZMUR1}3TVRc_*sKU5Hi3`#f5^aie3+X_p_=P|dwfrqck&eZtTjU11gQBPmaR4+PE? z;=da1agNhQ_-UIq9n@Jchn`;yr7ANHTO2Qkftoznw}_v(WOga~z%8t0fcxHB#=cR0 zIxY8Nz?%Zv3jDO_g;?2Igk@O^ov&wMl_g2tngQK0+cCIvV&EA&8LlOz6>x76W6fr0Tc)*L2#b z^O@5fn&4{xwY3RY8w0eotH&r)$wWC@<893`t@4POqZLXH4owbc4-4kGz`qi2_Sh+P zd@N98wkQeu0>U#B)A&$~5TV|HT_0YC?A*&hmR!3?Ac|*nVJyk3z)c@EeZlA}hMf-j#rJqLws{Nrm^2L_PttZ?5-)61j!>3# zG}`QfxOh}F--QyKCuDBK67dbckRMHtn^%-QZoqlS@$jw9D{7e@-K0V!Q(8nS@4}Rg zG!O!7iZLDP3e<7)3XW7d>OSbNbtjdM2n%%CJKrAmY5E`>dDYv)p8EFCnDU~`lC{n9 zr1!Ipv)mV5%RS{c5SVukIXL?|oV)tB#%)8;1%7S?aq$434l^GqFO`^5 zT|DYiF%WZ})M*$2g-jh5a)?2)*1vs;&k$gyoZEu>22Tg5c(@~Ur0HA?c+FMOCg4Ya zX)ZQz3%Oq=y;sh2jnx#JZYgHLxVNP_e}*s%wQ;naTjp0{Z;YJU{=Z4Rp$r z(jT^9Tolw*P+}E`i7F-N+~v_^I2uYxOq^ZG=;E(W}c z`_5~-3WA{uL)DaY5EF-`?-HRAPY!b&aU9XR`m5p-M)Zyi1JqZ{Z`|8Dp&;b^d;4P0 zy|izlIhe&#ByqHKZJJJH$&$@4tJt5a1L)M~3g~KA=T71ol4`bk0*>>Z2oKpsCfEhc z40NuJ@ZJdim}Ldd&7u$@;fEsF7F<$VB|~pAu@1b;z1=Q_gl8BYTe-{Gy9~U&gpij2 zfAJ2i;y8~}=ZgT6aoM7_$<>;NWjE+*d@UOjcyeg}>SRWr%O{ZVh_q3kIruK{6b~eg zlYqatOpBU4D*D``74>077Up@<;pDXo*F{=@Ch*cdC}{2sScg0VcpD8o=XTGZIQo`4 zxeMc5-10Buq?!BXRak-gqJtwZE`NruP?)shrq8r?+f>C(s1>8>c@3ISN;M$_r1Ps z)@B+hXIo$z9J3?UgL7R9Jh(Zne|ulOUGM3CvR?}S`beFZkNf|3Y+?QmA)7Gjo{ss& zSwQN8hikd2XwA9el?PRkuDg7?$7dde2SzqKtjp`8NZF%J+q(o;nTWsq5(TE|`5P z4V4ylFc5ASpYA5>2=q}pzUh047vnu=`38C07NJ;&o_3z9l6IW5=5ZJI=R2|LbI!_m%Dghm0@#{9j_YaSUeNNE znYp{mgxSAikNX?nB}Yqa4CLMcobG;l7f@&MYWqTw)^##&Nje4SH%k|Iu@Fm`ENbu$ zOi7)l9+WCU>)RZRRa5;#M4%EDf}G4v_nxWz}6(8#s5-DSxBC$vaRNw+Alp@UD|F zZxL6^-|%c`6&FofLp)Hnuq0`^1bN}-UgaOQA`eoAz@>vv$lpQQny9)rO^KZoS3xvNK@ZEmW6Ry4lsWPVRXU(+M-WBFu=kBiDS=Bk z)9%H2+{k$!MPu14RY=FK%}N!E2FAn}8}p<2;9Pnr*EMp0>epzrRV4l69@%$0uAY| z{TIl!`2yW;=TpYS`R)9RqeefZV;lq(Yzy_1A|6Mv1{=gBlsunVa3JC^^QzbhhSK5m z8SQ*qd<@CoCF3~ziue`Dp}aUb+~&CU78S=zWHqW%rG8eA0}jFD7nSKlySJ~+JV7H! zUe~QGD=b;j$C2W;>ljLMwxkxys}sX&#erpB!>WiURW`IHxPF3iMG zFmpstb?yEH=P`6F^mF--yV8I7;oEONeEjA+v+{TMMm~KwxSL=8*FQe`THq%?!Xj*& zC=#7gL9Eh^BCFbQT6(`##t6rCk1ClUnLd2t35VUaM9Jar$zlLIIEg-<(Or!?U3Q%3 zHOh+wvZBEy-2y+IaP6wGC=PPh++3c}ftI=swSV3TvlQVuA@6&ZI6&F?lRN#`i)pr_ z?-%2gBy|?fL7PJ-ut?WB2d`ciEX;Jyq}eik897M)OBj~ii9CB-*Bu?ICOP9b*W;c2Wjr~5 z=j`Hw4Hp4vT(*ATmHym%ycyHc+n8Vt-OAZBu5@@~F^*H0OKQJhW611`ayYwHw9)+y zG`snaV}5bXri4Gxv>X@ahdnE+9I-Eh7>aJ~O5fao<%mWa2aeQ9J+-f2#{}9%{O$XE zHwcHv7oXp{(65HRnWud5hA~g;C0JAt~H86&CM|LEbcwiMc9M7x&xI^LgpMuSk!f- z>&Pc)7jjFqJAQlGqEnmGZHoBC*;^adem|j?d2=ELS~P0})vT&1?%Qdr4#F1BVZ27D z0PZ}v_I%%Ll@m+3yX*YIY}FwbxJGmL;x;S~(qtYxAynoEOjW#C`v9Y8>hg^xzz;h! z%v^!inX3@?>vXimrr<7pFT}HxJ?OZ5v8D-!dRvPoO*T3e+s4a$@mCMq5Yxw(}rG*Eu_=Uu`_ddW00)W%AQ)fiW{xTe*1)mu4Zu?C-5+s95xDywr?jxIGzHB8+swK=t*cA0Pvwlw2 zF$>oL4#YTi5F8gSr|<*;LJk&04#W+EAPu=LUFEq8QsUxNJ}`ZN})X?`}V+Gq6;f3Bul&vN(zroib4q*=S?j<`NExJE0mi!pVOL*2Eov(_40bE(&t8oUpjK4U?BN_k+HR=E57H z@y02OY10>N$Tu+n9M+xPqY|B~|4)tvuX6X) z7Q24p%QLRFbuF`IF4itnY#10b*V zK$dGbPOQHNH?$~qAY%_t-i~6F;$n{EI%KRX*_01{yA@uL)+}HAE{afeWmVw;eSSy) z03hLqPLYsPgN!VT)fwSS<+ zf-9*I&aOx|t^5;Bl@E(2Oz&6As0G8J*YZIr>bO-0u;-Ui)V^fl@d@4$dIm1o#) zW-j0fR0_0JzKff53fy14hnJd;=lf#Nd7sZ-xkk%e2fh$c%{rs!Ne@UhoOnr&|q0g;T(Fsz_@T8(y6}lH3G+iF}QGV#*JN=oRIV5#>K6m zqRQ5Ku7ZVCMaU87J`L#9Pf?e~r9Xv-au6JIGm2Bkb?hgCL|9Tb6fV2%2TBE;?E$Nw zckwBobi!A9;D^B4vQl#xMZ9O7S8f5zlwl_&blP|DfTcjiGj=DCTRG+L?Z3(&6PEKi z^x|q6Q@?B4W*Akxa9${XEhe=r-h>Su$5xjF#;_i^ks<(fIg|FCe>t>5$7Ea^ftr6&oP>u_XODc3)}u+OZe+=57z>yTTU zQM&R$cJ^kJe%{4H`cS1yr!%!FDaO7l_*g=@6-n+DWxbl(B?5sSRjD~=AnezCOjvM~ zamhQfR7mjrde6NW@j43XWS~h@MSe~DAPg`HG4u*(!)c~Q16Rt!#;g9B9hCOF_|%4J zm`mSTmq5xMSDHKmel-uLXn}QIRfipfY@6!P&*)Hxa}etvs%ed&uJdMj6d`wYvQD0` zE+N0XuXC5L$Q!tSKDyftMsz}P0s1zkHa=@;Rc_(dbtNrxRfsTbmyK^e77D`j@D7+w zo)@`M>OAT`aL)#;`{hCw_r3$S*wy>n%g(0exb))AeUP-p)CV=4so1<~3D(wcedQ;# z+jROq9y|A@br^A7CZExg;b5<`_gbXC9r6KP=X^2otLyXz=auza8|cbBq0GzLyq{;@ zm43%)&}3#e?wH2!LsLPXu^=Zc368$P{)W{z5_v`{Z+d-S4SK=U%BhGNZ)>Znh=XY< z!zRiS!D8VD#liU5RGB3%r&N$9bn=C3iJRrZEoBRIIAM&OUB7|2nq)n1OOvxike$!JyBhX9Q6T&(TynmOvg*sqAG|cA+v-RRahnExn=C^d`>a(_l{RHLF_oD*!-Sby zHdM-2t02p_yKAF6E8e-;_toHs30QAj?3X*kbk&^4n?G(-+f6a^S?aBxSlQgH*@0&7 z;GsbauFkBVVFM*>OxwPcheLj($B<7nj&zA~It|l(rQs0c;dW^_o`SFlJES7n#F#~6 zo~$!q6f}zugD4L!V*J41qtd}Ah#|QW?7Td0%hl>@3i0Gw?Zv}dvCS=CvQ8Dg?+>u1 z-V((upD8Q0uvZ7YTFDO@^q|u;;)o|G)}kdK; z^rAH%T2?c^Lp5f|o1yUIk%y(}F0waJ%`QIdqiXvppsum40e;JIfb1L47wDs0jrzJ( z>%96DD6h;wAb% z%2@9&5c;@mLBI7;FyxTXYVL<9?Mie#>{UBKv1t*l-zm1>2a@yjXvdht+96 zo6%eibc9tCDav*nCPGkuWj%s`6TjRS09|Cqq_*K!Z3tfPL_X(xsRy9s9L6cccxd;g*;g z-IeuDcJr0=;?8V6w|SaYQ&{&D(xobwh9^@P_hFhYLYUfxT$r5}XWwJsPpG}AjVqhV zcbz=}DZRo8Owx-J^Jc(o=A!U;(4<=w$7!#Mu|T5viHB9#`{dP5}qcv+`<@xpDMx~VJIt~CW{#X(?PXBgz+ zw%T3w7`H^BkVtRS;3<9B)zQQpFRshlU}^?lus$sl+Qa16MLG$8#A+)P%)(r8qsARh z_5J-ypR>z~<;0R5kQC-j{S%Bn8E1p|amGzQkuG2-uqVwo;-|+L&$cy07wOD9o;2H5 zMQhYX<ua%Hlw-gCmjJ72uF3EXh#>OPIe+pLm(9O6OdyD|s<{BI-+B zDfv}8W~*9xn|JZz)l)KwxyXjeUF{m@S=BT}=(w0JyZ37Bk3!R0}hib`GI$tUO=*s1U&-n9C4&bNuvkGr^@ zpYk}sH5s}pS5-2UX;O}qhmqe;fKhAK^QJ&K>U5T}C;0pd3vOX=w`TW+2xNquday6< z)Uk~Bf^JdV#~mi^+F>E`JOW-tdG34az;nd|KZfm_y$*Ro8y@U5X5l6iPQQe^Y5mDe zxT`lI`aGY@s_f!^P+mKVFbn6PU2z&j8N|5>`Kw#qXF9ZwuH@|>ZIF(oqy6S01SPxZ z8l1l3cro79I_wthiw9h>v|m)&g{&K*Vu{+YtWCLjLgA3734L=@_X*qyg8Z3??{rVb z;Oy0xivi!ZIZgaL&tU~WWiZct8=^|YsaLfkIj9d)VhT#S2!y~Bl!+{O$$(A~-kGS= zJzsm6FmoKJ?~lEBWyOO$P10$}`3Fj|r2gGg0BOx)Wj#z~ud4FnTkyI>MWAtE?92kmSE z`>jH6jFhe|(CJNIK>gC2y_g5PZu{dd&f%hYWos6BBI~=L&jyUEk|y0! z%(17axs(Vf&8=gQ1i1Z%y2HT|;_R2^DY|NeE9NQST&z1bg+oy$-Prf*mI*cnahE`+ z5}71m6x-K^GJ zR6Y-w%EFl*Sdu1ZP8V#tf_j3ID3ca?><;|$Nq^k=;q8-tafc?-r8;I)G)`lo@~r6Q ze$I=~+bX&Qy;quIG|oYAfFfrPFFo9M(yc%*Z!_mWhq#lkb+~wV+IBe|*Dl#82s_8h3KJN&k!oW5X~bjaC~a^PGoDc9&S zt)?x+!8lett=2jY+E8`Dz<3Xpd6i2ckMg0wsS8hNzkd1B;=Q=vrby`Thlq2jj_|r; zaG}E3SZw`R^lde)v)2}o(I%X`6o*cYJ{qXI?i+(O2KxWlGbF~Gdvt^D5(d8w{0k8t zPml0d;~u1WkQZ?p2XRzYd7Ex=}zj(fu24Sw_d&ZWi>#}56i`dI~S$j(rBrH~D6PB5-?bKmHKH*}wr39}bcyGqG z+q$CTTrp$g7th#$vU#oBi6&v2$6MGH^Cq$(28^kWG;m{3e=^YW#FXy18gbvQVj`{UeA3e-RRGp>3Y zvvt94E-?e6=$@U?i;G*b`7(P8OTvJ4+qxwKXKM9>frfRKhA};;M8m=QLjd6do?r$* zq^&Z}{l&g4BNz2$FTUsdO+lH^u8ZP2VKwAsAFp)G(jcI1pp1e}5a$5@K5S+bpV61K zz?5{{8{^iU(g%qA?DKxLrC&VptX}7|&a>9r)|92;l8GMrFo0x7Az{h!$M-CmiSTU4^4Ax-`*~W&tD#M~`W)287Or7Ydxu@(< zCJxhfhwvw`;`LK)K(n%N zBIGDOp|cN1IPMd?8y&F3mL<1r*!LbSJzbFm$kj|)HAQnhl=aexWEw*3=Bg-z2(X@( z8u1`BM?oBBGCIDI2c0|a>eK2p%1%Z{2)ilzjZ~cN<*+~R;yg~Iyd5gh2U$9{B4iWG zd%vY#qF5**qe&+jJJ_@`ko($?eGv-%x>*SBuYkY3@+2J>oGZlruLr&r7--~EuNdms zYdkOZE4r+G9VB(JCB508qX&T@XW!su52_MK$N387o4`vAF#Gvdm%G>@xfA$1V?2E~ zi3c6)^fh%!{YEY>-?Ji3!YK*bHW^W*Y>Vb~U5DH&U{qmbSb9(*I?~ZU!`Tu4YV}|*5jhOqzUs)=pqvZtwQB65FpAo$SIePOcc|P^ z-_eY5`v+gYa~2}L!IS>cKJC-wjy>M9`J67TANlf-Yk|kMSwj!9G-|6b6F`~Y~CD|LX zdA4rBxFC&|9z{3-R?}v*=v+8vbB@&L!4tFut?j=N{6gDSZc5GS1jKnf-#<{S^*44D zo7GRZ>)RTXeeO-Ih{CSjwgtwBr|!rre6R63lL!@=^63R8KWa%9M#(C*R0>A}@=ZgANA1am;u7821@$TpL9I zzp*B}R{&wsD3f%)e!JPbxYSw=oNKUj8QGR1&b`$pkE@Rqwg|< ziG@N^eALJD9vYs5mbU1Kxakk2YaelPZ|7>3wLeyv55*qT=b)i$ao@3D=Z<_>q3{^dFXqTA>WzOyiAY~<;gd>v@xY&YJ)k5`#5M=1Z+-= zG#@-A0_ArK3qu4!N;u5rT_yw*(*Ahvo}~T01Wvwb_b45AH%WOI^-rbh$i-)UfmY-m z4os<_sh6l4`&d-aC7FV4t8>#C3=eScOicP{zJ3qf=-O)BO8QO{#-Vufgyw3O?l53A zpPSgrLuk{Y4->y)Q;>#P-sdb!4m}uRj%ak=>ar&!aO}$+BJPD8-l1K&L?V(a_9DsE z{;p9})a^2bVhi)F0ac6`&5JiHAvi?wa_F)iU%Do2XU{gD^||X%;jn*D=V)DJs^qJ(z{KD5a;6DD3&7gHM|kEgbuVh@Wl92zg`W)C)# zQVRNIaZ)&gA19!23YAU}Hqddoy3+I`njk7rpqL_pui&@mOU;7W5&7p^1+#{-Ay>Wa%JjD5MSAWv_MeJP{z?6L3Fb_c1b7Z!wV&gZopij;~auP_{z z08;^-Sx^#z=(W;LA;KDwh(`Z+U*c>_GE}&ea!tepUNb`8ohXpE1klvst=l; zv5iyFGjrJQ9J<9sCo+;h0d!)mivVv92hicfeS8=Hv*7@}+JAB`fCWD0{Sr{MdfDP( zH|BFv%zRte!MunZxO(8a!U6UShjg$Y<6BLn6Xt zDyOmWc{-~x5=*xDDUO+Mc*^Qa`p|jVN1R+z`vPh7%gsga1^u>~3V=&aHmlK(yLh=` zOhub-O{>##7?5^-PCdR(yTf3^$KG7HH7{^*Il!at(f4F)%*J9p$z1v^y|1J%eeWAiMBaLVF0=&!DpatC1v9`(dPPMfwR%RIALSF zi9(v`0M2QzB0#oP-5BL{sQN5k@R!g==IjiLXP^!C(7x$b0pwwilDDR1SEHV!WgTvL z(Q>beGd`*&^ytzSVH~qH$_jH@#*O=%O4zMRo-io2VJ%3-J7W|5zB;EbR?v%gry|}% zm-uU-c6<1_2xsIl(PW9H)JU?WP1NyJ3=^)0a6Z*gq|b>X~o%z+?v9lYR}i;GLM zEyt9Pv`jiawtW~ae93&@6HOec-B#8(=9HVBlFZe0F?oXUnQNoVCdkdMs*Z2$?3H(V zwH1jMwvD%QdK8@tcrdV&_h}^jj4_Dy1@6@Mup@8W?d(N;$Z9_qswu-V^7@%|OFHT(&-?H|w8kSYn9&ar~1uct`CRNOuYJz&Krjdt@dEBWvFr~;TqYrLQ-X&Gf zTpNX(ylEIs7&zo)tvmg=i{~oCSPArtc^yUI*ICE9I?V=vZf?dHO&1c!+=XZQq)Tuf z4DSy4O7geEyFT5zF#7o;zI@zO>Mcmwu(y?LbaH{x;ioFuT-;dC>I8tNo)@lhR(Klk zm{&!#q%>ZOG1(3jc8Uj4L&=>dSVS}CJPYU-Jrj8X<;5n_YGG^@3Z^DrqqU6IJm5$cAqda4`4QBRm!x(*d0s81LbG1^fNSdwi8Dd!^)yxXP zy3vobb?TO!XG3VlaeCN}!A~{D(NgH!+x7hXf?wmY6(~rAYj(VQDXT#7vMOdoPD~Guo4u+BDKt)L8;t2y4;#w-o8M$c#u5(8I_#&(c`PEugF<-Z) z$@hjjn;R(TQiQ&sQ(tA}5DZ3teP28|iX8w*DeTwd`hQu7?YA!Sm%ZncS6kV|H~AQf zg%wE`Ry<21u6h-DV-dxEy7qw=)1%z{Vbxuqw4VehEi=@S+CclZB`EGSAN+_fJ@&+oH}h0(L4 z`o$f#Fs$Y%!*ZqRu$1+%7z#n!*{sLpy83H zx769iO}4V>#-;!+;=xPWaphIKsC5(x4A8gpLAwOq!vGB&kno@+*1g z2zIt9pnl%fJXSvS#ms!VH9b#OkFzq*v5&Sq&wOTDh`18p*e2bD!k=L7Of9t08Gok- zLv;@MxKzX&J(!E@aW9DHZW5!K%RKJGU>?hnhh1Dv`7~{fI^^qkIOaN?lCPv@yZQQF z##=z0zNbmHe>cc3n40P5UOXi4n#^w-g?ZB@b$}uZHULaOv%l;TR2Y^+BB+_dIsDjt z+tu;$2?PnWQI@|K-t~2Na-DW@aG&COTw5!?V^a9tZ(^2Oqz zHo@jDBa}ZOqBC37ndG-e;>3NJ^<14+7r|!`Ah?Z;P2N;?(zl8@TR51>i^KV{ZgG(wyw9fl+Yk2wf2%axm#MGN@4FcMC0SyR*DLttux-Nk(Dx>9Y+6?5 zMVgy;IdQ~65X41Yno;|Oncx<#?DXX*ehlguLT|bwa->+5T0F!9>rae?&K-^D{PcnQki(w zl)uH8qw?{Y%&}Vw7x(OzG^=SmHgv1N3r8BUu=9$TmPLaZ@6{pkKhPv4cXg31pTL-N zxh>$~Hh&7yIl0)oTn?^ohjh;C%e5u3!h@ zC>LGIT>Dr09S$*f9`T!joxpJl_b}ua$914<-ga3S^j%XmSy@$cSyP%WMb`K&JID~c z4g^2lZjtVz)-d;7uihC?jQ(g?j`H8`{6R}9ugH3mSlY4Z6EesU52^jWTTd=;hxzP)@3R(KGkay1}!f{ zz2%`7(i$fbo2HJ=!GT3eaOgwX`n~==u7yoSI7-WWZoonuon$!YbLaGfEqJix+tzF< zoDvm7!|0&KX=F1@S&#-bRfnZ?E)NU~eeQ(d&#?Ss)CT^y)jxNwo6{S0S4+&4{x+_4 z(kH=I@^CJSb()(p=gBZ{G!UjqH^R~Gjk-)yv`@1DD_OhC{?|IUlpCGf7Y{%7~cs^6d#aHttyRCkg+qfa* zI|4QP@&vd5S4?>1#R@e^1#>>Ac%FkGN#-=8RnSiH7(oldOH45;0*9+X-`2>)6Ldt; zh8Q@=_p}E0{w+S97rY7l)e@6qm*vycsH{)Lx=bB%nz5d;VQ1Sf#;iuahX(@}+|~Xa zZolk9#{yLc-!bLV>FK!z-^D8wqF0M36c#vcL#d*&tcPu@;x_V&G$>Znt%q@AlDlt* z3GxINHfqDz0dEdy_Q+08aCJQnXf8&)QYCBWrovdI6@$R{IEo?0zAHdz+)c!Fh&^2B zLg#Pzgt+cX3vnHtx9j_T6Os}4RCOA>INL8~R#$00(xD#KX<5=u=`6nqv8f_tM~_Wr zI_`n4atI3h`%ws%+BtYjy<)HMsPp*3j2Y0cmai!5V1+OWTVz?g@><0S=jf|}(IBZz z$2~4wJm}Ool=Ej)cvM=b@N};5PQw1O`-<)bc^eYEnqVCUkc@l^!Xd2E8XBFZm&>wd zVwtwQ`Nf3Y;nLMPD?XulOKkv8f2o&8b+SSD*<#W9xfWV^lLvLh5vQVDc@%7k7`!1> zuAc>01BduXZtgF$FU?^gY0%Xx66tilJ`^6xr>MJNxu|PKf)YNby5^X z&;~(J^0B9WH5TJ^T!J=Q_|j!4=z~(Sckyrqu~DtTJN^#^fpS~}|ZUJ}RYL4ELy9PZ-E&7Z{9dQ;z| zo zI!t55+7YWfj*~hFYgY7SII<GiEk#vNuMWc9UA-!x*mE{Hb%0xV?Xj$9g!&7s49kqt{r4GtJ9e!Z7d0^67zP+Gag=A=XWae$A$3NH`hEEQx9wHeUjlugVyG(DgAfP^q2_YfbhlUs1AjG^tV{T&J-- zpxl3eD&@-gOb1ti9R(UdujqlsWyHPgf9Me4NlcVnrGwsLcSN632 zQ-AD4`4$=0$!`6%{pgSIwj22J*WnDao!hrgY;Cu3gCttlWN2vAEme^5BhY>FEs-YFWwwp5M&?^ysqYmTqe6&m!-KIjEmBVMq>1t} zffSKk=5m)Et1ho71mP1ZNC)K{^S@=jd%DOVXa#u7hd!$)fhlgFuW)OsHJdj+!XO7) zB;z(rW;sWLEAS-r0`CDeT;O0h$G$bJAq2sKy|`_Y=@Dg=Y-!BqG!Zfx$Wk=>meu1i z4nt?w@Qkuc4X;aS#-A{(;pm7_!N{wmkPr#QTRG9p^6kz5_jnbB>UlS$MLr*KG9Y$s zwocisEtIU$rggPVUJ!>qr47RGYSwu4FCZ*jI9G|yzTj}0`5u9TB` z$Fh;p7SrEr*G~xmekgW%tB}{f%@OkSMYpbhGNkuL^D@gr*shezHj-^#({w7cj>u-K z#%VtIx^``jX1DGWGn*H}kOvPq(9ys}uTEcO`u&IxxOp&R8Ed-ixg4=*lBQINZrRte zo6D$_%b{DI|~AEFD$z1O+C`mw!xmCkId*;ILqIi|%?$gJ%e zMu1_qbnAdi0)yF}ai8-G;GwRrnD>Mt-PN?6!Z)CYz;6fLa(mYDEE(q{-|7Zcw@ThD zO|Gx28k$rWa>{Ja$fZ5w{LFC0CqQ@6le5>Rl1TA(n?Yu~KNGatg{<~s*rvMRO*`xe zYHoK?q^jFY!VJ3xIg)^$etQp;>BLo;&geG=vLVIWLAU&tlPFfpS|3?1NW@u*(OMPj zRJRGBs;Du9IW4CyK4hm&w)a598+8>3#Q4_*`wwr)+c&9}Tk$AmVk;85m0P(NwCm)0 zbVAmw8k#Jrxal((TsP+$vw%LJQr6F?o479)yaF6N{5&?o0Nk>&VNDmT4=UR>k-aKL znv1;NqE-zo*`zXL=>g}L)w!-OJx1^Wz+EH&u<%7Ye#Sq2Y$Ix59oogZ*ydu4tA1Sb zs*=aZM42@*pNr)fAssHRGMEfrG~Er7yHT*35&O-)_qn`8x7pZ!?kR zVw=i%sOGE`O;#%*YI5Wkto;LBD&-~Xf^&jB;rLI2TXyO0-*Z=ozyuT(&f=A$U=cy}!#y0E+ zHYD43x{XP}S{&=V>IAfD4K>HAQCC96)vN%){}~YJI1MmxG?(Z2`{q+;r+2scl>S@? z=2KIVFqw&h)-(~6cLNGi^;MZoNvUwT#Ccz%fX;B|Fnk~Y%7u+6r{9DBBlQt!csuyi z1^6-&=~l@}#3`~Oiu%0DmT}0Zd{t9shJl{;p{DbgF3mX&9+=m=B2;n8UvK4nO7C99 za%#EjXKj|rDw^|ZAJ(N9;)c~(x3jI>_oGH5<&Tq#$)vvX1FFyD6N9Pz_Z^u z`RBVtk=%arnTwNOr;%dA8V}77t1?}t9oJm8a|?{8`_h@$e!!@i`|b09QTGKObwHEn zFUI$$r7pvJxpdriz&j-aQq~9_AF7+8c3(FNoBFA^Jn)p8ism5jA#_iSXFfWE z`n3|q``C2u3%;yv(5E%)kFBhZC1){f>ZEA!h?{n0qCB>Qn>rX``qP|sH$I?(LG)c< z&R);)DI?z9Z+c)^{h~w;(^Anos?~n9RXWW3F^)uc?#Hkt%zV?mi+?aV2}TgOZ{rwq z)yDm@rTe$P{rS&-`s@Gt4^#Ve&X9tSUvu;C{tNt;cXlE{A(}jjinPpeF1I2ZQ07(sU|F$`z&cQS`~Du zmXYU|jh&gzuKN}AKc3dSL7Qbhb$ZPI=l}lmpMT4!V;&)&M?C)g+d4tJqgzj^yoe|! zGL6$&mXryWr|Y`ZBD$arS3Vun-vML~$X>WBkBZ@Sl5NY*Zu91% zFR+~kneK;zd1G_4cZ$bs5?Ir|lzX}*nHd_OZ(vjE4iGTlpU!dKxg6k+8A9Z5&0O0JOg`u`84d$rDE#oAhodlOx!HimG9YP^X5z(Sw+o~r}? z1>EO~#-pG2b*}NA=Wf9~X7ygmbu5Hz(g{VL%`D2dCEc!gUo!)M^DyM@*E0|}7}N#c z?=^K8Ah*%f1-A3QSaWPLk#bm~Rdtx}WxwX)Xn9l>N1B-CqiAeqDG<&+^&xz#siVXN z?%_9MyYq4neH#WgfoHkqFT0t>LxIs6O;s%F z?nq4$_^-W8&hFNO@OICtz_OwEF^mNp7q&takNcFaBpsHz-ua>XJ{S!{@!%5XiA3MS z;R9)ZKwMPr@U2T3q11vqU0_)rS#c;@h%>Rqsyw1~i1TPD2GU2z2K|+D#R<94>_WhR zJuuD*oRo6_FFUq#j&=9yU4NeCLoT5j>!{21OTq~u0B%g-7R|#&~l-kE3^-^|_u?-B=c!z{)!ac+qEA*Zka~jp z8HD6C$#;LkfjLS830@N`e0u8qZ+^gS^D5-hwsPrnNv5n+2g|!`UM5*@Q$2P`ZLs8c zU>1E(OACE>ug)M0Tz|jO*{63>{LbOB-`>mmWzrm ztM%}zmg7vHG<={3cTul}-zG4hMce%?xb0!v+^q-c#)HlIp0XDBI94s2sT!Ah{$mD$@?~eV{UfIpfQQGNG^OkU>JrUj_lql0U0>hx`yygYzX|& z0n^aJ)>VJ5$*wlff;OlL*}hX=D3>*s(;lDmoPA9^m(D~>=}6e0|r;2Bfo=? z&~LH#ee^A4`i8`^ePL6?vKr@gsnZBjDzE{lT7+H|p!k}b(1V%x$UO4tkp3;?S?GvS z9U^{@yvJ=MeTHaxW`C%W9eY+6qF8Z)Y41lebbUV}@B)U-MBcQgMIa%A2V@f*93;1g zFX9AFYg%%*_m%BXSJsJYDzL88O{CpClD=oVm?d52auLIJ9o=3<&td4(PYuW4S-diI zFs%>y3zhqMzKO2V?;hrH%XNQ#j63V9oUd^^M{x#Z7E@P?vd#FuRR%pee8AQMLKk!U;AJEf^?iQx)oxn{L5uFl=oHhAkg^}Aj3jX*)_N=2p+4wE-hMsl z!5P{8sfpz~VrPcL8QNbhVyVacklZeUunjqlN!2w#x=u9f$od+Il4Qd*+a@@U?cz~u zsvw95O#cILZ(u?^2PCR6{AP9w)W3OlYk62d(jywPb#5Wwce%=#_i89}h(_$2k(Xws z05u(oD5C*I_oLgu*#Pfr#p`*v`R8#0za7w7279W>)XFiRhHg93rR8-5Wm&_Xq*GMK zM(!8mzL^X3!-$W<14la`uBx{1wT=-9xn0K?+P==Y+F=8ftedi_FIakHf+zV>=dw#i zlRL+FNUkw40m2h1=fJn$iNEZ_={a({wOF&A`bBA5u}ELw*)(>ltCP5ocP7fDYh_XJ ztKYa8BmigH8IT9Ma97C@^eZ3YUsGez$9pk%;Wl=(V_v~A$vA0gBnLPKatv*g9dnkp zrlIhO{FNRHjQSsvYu+P62izs{7vBmzf9_9RN`Cc-+l~XO*6JA6K_uy@Y9>Sp72A_> z7@~A3E4+9RU|?EG65S((4;*pk#!`}cFNyEo=R1_&9&XFTiuhtsK^562M|I z910gQA#4|KIYxbR&_aPcA#otoK4tC|&;j`R=Lqk;#w{aF=CYVXG7aM~*7MQm`kY9* zAG~AOi~V}6nSY^263Us++_R>Liz?0Y*h@Mr(>=+}blCIrEUTCDWTP`d)<>ekLnJXT z>W%|FlkG&e5i5B4Q- z(rs)It(e3)mOJCL=y0NpVxO^mX9F!-B1{)r|5_qapAhanU^NZk=ik&C)`R8J%yVDRRm8e2z;>X^M2oa!xB{6g^WE^xH4S0h^B#zi0gpJiIPe>1 zGUDUsMdI~Ju*{x|2P%$r?#rwxnw74{P|Iu(jpEqxBU{Z-*y~$C&WoOU?l*HDbJ6=F z{2o^tJR3y<+BPv+vh0Y*B>~uTLJ!=sr0I#of*oBPwHIsoOK2!yh#VgvOZ(v;?+f)0 z_y+j&wU3E^8RvdJ3$Q%X6+E6GkLTz(_}tKXT*q{rdCC|WX;Y6EuTlT1lca%e2_G2r zdGFDr-%?Ti9rx=hp3=bg?i}LVhP*>jjIt-C=$5IGC@QQgs%SGXjH|W2SQRm5n#wau z0o^a+nFEf^3!u;X{v31O$+@tObX3&&aLD%PlJrb-=gV50?E=Y5{xQvIo z<>&qr2{E1vtiRVkc)F4dEtKrkas?-%ecT}{;uM!nBt0Mc7*$@x4bxS8(wil_DSk(w z&k&>`bWy+Y7yDQW1cQ4|xMeQG-Q??u)%)xI{epN~B;DHj)En?~IW z8TpVG{ic}J)h7NM$8g=7QV8zv%Pr4vx3*4`1FBTYxXXE6M{JDLDvDJ%=W$nFLPOBI ztZoA2!H1!qZ$0Pw7k}LC;9viGEAHo4HUxGvC*D5nYqNCaGDSt#iWFkHCTgu^an$|T zGc(om%&s?JKXZY|M^;tA$yI`X4!#Ec{7O)y|Mg4oe*DrqnX|4y|863qafjgZ&it!F z&bE>8fQ0gVrgNNCbG~gu(uthMc}|&{qCuDwnd^y1JQ1Pp!6)Fky6IohGwA6}NA3** zIzP^`>?!Yuv?|!7(nw;IEs`Zg0@&2UltyK3vNJp|Q@cY5e0%_LbU?=O|FKXgMlPl% zNJ8r0Jf8EvSkCE>cD0tVN{jaHd^b91W^j1!$W$KRH$x$WE83eYr86W$cCF6cfz=k@BOEWbNxNn+P>b^Vh z9+=z_>c*z@1=wfFk3J#y@-i)>9p~|!({^Xeieui8dFpUa$8;^|TBK=tMf;Kv#D1jY z5clOi@tIS+jZu!i(Fmpd&Z)d*u$OVj$G#xVz7>0>X2_zf==QBwLkoLk`q+8Cne$6S z>V==oGc(We+Q0}q9w5hvkYcNJuEX7>c z^$f*gY}q}2Io)_3dG-LMhPxW#_+I$hyM*$O_3dUgXFCF_;#3{GD3^Jft1=O^n8mg& zN+wwx(ZNjq!b_Yp_lQRyaHR5?gUy={FCE6oY1Vjm?QXeuCqto1&f9)xW6$EH1Dt1` zSM|s#?*^kHe^k&VJZM1iBs6pg|f@&DodQT3j z|D!T5>nsMDkc-hYy>#BbMt}o*AkB~Zj-GEMFQ4x-Hum;u3|dCLE#wi=C7;BRb%~1A zR)ebYy+SH=$rhU~vUmv+U?C#$_!HTkfjhF4Pt7UaZ?~ksnwA0lc2IW`rM>WT7sBgW z)~8<#>j0b$>y9N}chSL4l5HfIK$>)8H8pXk#~lU8dAYJFYo`}zy;0F#yb@VJs?j6?IPMDdR9&=g_bua3f6fhlimqg8~nl4(F#q=peuZUHw2 zIQSzcF1U4EXBllmyJQe?O$MG%M_Q6GYidYp&Lg(>lX1A&rP!ySt1*0F?ZOCgiuH?N zUnJz?`Ro@@yk*p(+15R4u^aPk7>9H?mV+<-u4b4;dR)>*epY(FP8(smw0*Kh(W9TZ>cQGS4!V&9Pxs9z>{0!J0sh`3zBA(|yKQFbFa09* zS9j>a11?Cu3!yW-WL8jJUfkN5TVBj0Tbx8~oK7fds>X-|Vc17%X|u9gq`C^=jC-9C z-Ise0EM5uooB?h0XlD$NceDP@GR{jicAF}u@?6ENYT!lH=F6Ojs-i3w7j_(Y7j5h5 zFU>x*6Myd(_gx%m@74UpzY^s?t=7PH66DdfEN-@VTzK6~Burlvs%M1 z(B>>;e(=BmhC8Cif&YyO?OCMX5c+Ds?z?cIr6Xo}x9oGYNVyc_B*~ygQtQ9!qXbK45JXFyab&{4V&_$9uUn;9J(XW^5fIvF)@aNz!AnRwc4xFZQ7t`=T68 zZ*gvfgFr}pkNNHoJD>xNE5Pf~dPv~rtKITLUZ{E&D(<#^h9*AJs*Oosj#8xsuCk(+ ztDRr>uI$I7_Q2I1_|DM&svTqJ?(B1w>!lsDjrq#@t(%jMd`7gG#KDk@yXdCjl@0%)Xi?JKU5#>{Irad)uRa4#M z6)mpyx*6N(`-~CheyO5w76*Wn7eRh!y4Ts9-OQ%ZpJ(|6FS}guswL%-G)1gBSQDBq zO?F_**=R<&g>ZGEWmMxu+{b&p5$5)0)kcEv9ZFE&?)Q&Z^~YK+*thz~x@AJ>fXIBx zs%oC539~3Kk7~F$l$h*yq954c!$sPAs}$zIwgT{*U^hw`!u{{L<#Ih*+BPqWIo`H7 zh8eMlqiiDu(&?Z`jrRNm;KGj8r_Kwhdp?Y=>>M3-`iz;!9aQCxrd2yzi zVL){>c&Sb%xe|@{^wIe)PsTaRfIHULJ7|Lu7X! zac;SKD0EX3BRG27Tu9P6y6T~tg>^UOtiA-!FMPNxxiz#rJHAi7whhA(?H zedSO>=7dViS_C>3YcoVZSUgrwNNU2OwVQAYzW~R1s2q~k)y|7KUca9ph z@YN`edd%aWiy6%aJ+wTE>yNJQtAtRl#4P2}$K$w_ZAPMDp4;8XZ)G8vKBDJZ$@9Gj z(wUJvG9w@_H)<5g&D2)gTIIDAEb9~1%jtlT4`s}jMP(rOa+%BMvQP^z(-;qU=znT0 z^Ugwz_zspUFR;e*VfmQ4?iA`&Lp(yABvQ$0?zT!)wC>1Iu|e^5>Mu|SCNqNgNcjUw z72?8kf_%$|0npEock|J9!G{r~%tr!b9FDD1z2YLruIny6FnvN0zCo-LaD z15N{Uh5OY^=toqq_xTLQG!wFX%4v;jyUD#P(_^V4G9FXLqm>MKA17NwOfrmzSEeHb z`f__9gxl40op|uFXgf0w?)H}@W^!Y!3twPlQ%}HVZuV3mv!TC#luR6Kf{nxX%bMEL-cE7oVZOmg4 zPyHUl$g--~abMbwA40YBCXOQ7_TPTF__!nQs-UCV9FO8Y!6cspG7D;Mzn)3G18`r-c4l$s;z3+q1qRA%*B*m=uu{3L+I&pE_{GA;bM1A z@Rh%}e~M_}?mephJj;1aNl`8>sY}`6jG4;xfyFhJ&6ww`k{5%}i=^SSh4s*X!doD4 zv^@V=$bI1h%pk;@rHQtyx1-qPP_2|7mE1s3#S2~@`9Rk_iH~Kx3Xct!?=T?zT(P+y zyi?ynTod^^He0_dzzDh1@y|BWo1A#rj8T;i!;wwcHA`IxeY=+h9c5*NDf6H8*{^PL z2!aqFV8yV|#nAPIaqH>kOX=M!RLcXmIO&RQ+`4${r>#hc%pgsMtZPoo@05tk6y}scfZ@*wn9Z7IZGlI5w@xJs8!Q1I4a2&vdtW>6Bu&I zQDbM^Qs6#y54j^Z_n&?E^(tqS!@XCz@F&hGPX)}?280pZ79P+twyZRh8bq?K~co4r+83jHdF365!<*M01{ ziefV}IW*O#oUrgi7qEB85tO^A+(LMDx9cb7hLmjcq=%MCQ1dZVP_CkxqE=UL^C;w@ zaS?fe`|e0iHpW8d^uwC^}kVaJK&Z_Y-!mpGl8-^CN;!W zLWE!|9=TA0q(ym=p}{qHDFg-(JODU4*q6OFO1TYk??@h(8VkjzY!X;)57s~Qy9yL*GG&bBU91$*|B8x zo~b2L&9qbftU%4Ba?S}VS) zsLklWS=J;+KlE_ntcT2K#vwSvK_39z3tRy1y%L@=(B0&{V|29LgvZ6)q%<2-)@EGB zQI!|*8m05n?fFbD7LJUOYkmfVLGXa1-y5aco6ooRx*|X6zvXCRQ(x) z3vwxnGO3tgG;YRO^~_|ddl#0APtTGqe1Om9uRS7w`N6He0k*TQ zlPOkJSFHkAvXHqfma6H>zL0%9&>pv|$Cp8PL0x7upFOZP;|`XjNM71D`lnlEaPN{7 zZF63rW0>}eDQugJ>`jMdM5cl8JU-GI5>r8Lx_jw58j+#P8Ywx zw^6j&w%|UhVn~)!)oI;q!;*D%li^-2Wk$J**52HenZ5ypq3+tAaPbRVyy57*KI1@< z-+abxH{_eZWa>z_#5+B#f2Ix2r+!%jMLBs+W5V~j06-d zoN=@~ayzC1n`MqWN%@p!BI`%VtBOsm=|tI>?#rGTp29sE`shEfF7sW@VA$(g9SOkQ zJ5v35mSqUK`W8SZI#feda+2g@+;*!h(`?*$Vc5ZNa&?G+5OhgJ+}G;*jd27z|4vDa z`fEt;AFf+|&RazzfJ$Y_HX?1SwXM=aV2&fFWhO5ZS7Zk8H1OaN7eC)g%^tiUsq2az zK3|_1{h4JFD_h!Opve*)ykD|qZjm?XuoNUIcae$29KgVpvP=VAy)pOCImOT&*v4WU zeRi7h-n$qNY>No@l?z&PeZJQ=Iy&sJ+lzcI$1&lFp%@N4Q)6@*{+z1~&;1^S6L;tO zWn7APdlZ&!#37D~EP<$n92l!pQX@S?RW|WVd9gL-u#I1mu#g1of#+NQUI%!GuM~uV zE+6h5q+8ybtnHkxY~7c&&Dhb!YKdYf`h~4|pKqHPjDwV2STCrr@h7bNDD~WdeZ9v$ z_v>mDuk~$mIun1XqjPEc6ecN8l17ni61wqHs)cr_BdE8vgN$FywS5*|y?4WahT#+D+K%j; z=rG;A*4t$5F$?Ub#5+-LjSrQg^+^hJ&{N4fV)b) z@U_BB_uk>{fLngc`?;d?)C+9;R)Oxj<dI#%L`r2Qeoqff<+fr(o;8r!2?AX>xl6N3j zO$#o?P_iS-)SNe^*{qof3WOt);DHeg|Ko&+_t;K#4_ z2mPP^ui*Lx1}+3dm6=65%+TeN5+;>k!p%^6LbH%ZslP>N%(-4i90F&IQzy} zf+M~8{^yqYu<1^#W3DSXjwz6Yu^1al7d?h^7BFYQ+6RaubJ6NvM;5FE~ z481$Ep4ztLR$@Gm3>cRKmffPu`+Aa-%kX-D7f-@13K9H^g)WPBYGOk`< z<&2<u458aFtG9`XDK!swllWf=L!?Ta#>-i@~9Et@P=+NUWQHrA>{#PiW)DPw&u_e!zB zj5?$qm_u0Ib5Z^T+77C4_NDjJ56M(`Lu~PH-fP>wFog4>&LnAy5vy)4ilJ2zn`MoU*qu)4o>aXF=Zp_U3UXAUvS?Li?c@gDg z<4qB#Q?tj}SVy#`CMS*?4>%1qC>}mBSV33k{Pidei2lpF5x3lo&3l(tq8iKW7_u!% zajrz!=GCzX+E(#&k(Qfe_5k(EPM$#A0qK>om##4SYku<x12 z?K1&~%wn0SXeo=b8;>%@^#V2Zu6pA0?1B4UO#em~Xnz-%0s3?^WcPDgxo`Oi?_wEk zW0v4p9sM-QM)G3a6aAlcL%Gq$vCdRtV&GZB?=PkfO&ws*io1p{V`fCo0=L7R^UX0nMu#|_W zraX?;HEq*0QgqF$c-wXw?^`=HroFmH%%pL`Jx)IrDZIr~^E?+|v)8Uar|r+(Xj_Jx zj}VVZTvZKe%2H!o9BMAIZir+r+Ilexb6}=80nW(r-~qzA?*#4F0rE z;!sswPs5?sRpxaaWjx}QI%=%4w9F@Sjv8L;wR7I^M8p{j9grM2f92f==aX{xU2d6w znpQDs)>%Y#&6h&ea^bv@s$o-^ZA%tkoSiOXa|FgYbMMp`xXQEp3HTS;2~?LXHxE*6 z$nS8>9$ec*UeC3JJ-!<(ocggg5=NOBM9BojhU)S z7Fc#(CTSz8y{SY`ioKs3BBNuVeRs@LA!Q$#9hpf|Jckd2F!w^Fz<0D0<>9NeaUX7< zu~~k{>!u#Ed@0Lg%j&*ZMHbNll`hcPGB#s!2|+TC`We$1kih-T&Rpou_;>Sjpyci< z-S+rA)n^%Lnj}>`&TYKOJy%7ZrkkAGE@>}Gf74)$pe8a%yWr6|7IgYJ`YK}AaA+VS87^> z67`DN5+7-j(xffQs@!-xnkBTdU;tbvvLS{o}o-@n)kqOJ+}hZn_L9J?TJ zM3sN{EAvApL+@tTKK+epk}9udlg-wgbKC++t496;( z_ezR)zAvwwXZ0Q8d;aa^gl}0UCx_!GS7^&+A7tDLsk)@B>XNDz`hhei=Nd15l$@T1 zl>C8s#vpVvMGVmYCc-W7Z?7cKGKVW~mH}|Wc)-1exNnJE#Y##bdmQi7RKkXr67m=I zxd-lb&&-8KNkI8)z$x+A&DXwdQ<|;MA?c%BB>NUEGv`u@o@HXMs)8+bXR0I3NKQo1 z_5K4ER2+R5Xb0aMZ%V_v(Y8EI;&q9Y!mf_uf_H7RA*&@zY8EefQ^olOZSxh^=bZz6 z0&NHD{(#|Yv_0=`?p9!15IfY{PWw%=aokJZW7_wyX{=Sbbi6*e8KWMUd7Kb9&a=V4VcA9}*Krh#EgE*@j%qW_%Le`q@O zo}(Oh@qT|3&J5mSm!PN-tYG zxFkGr9(Z2(0Da2?2O7zriC$lOK>zSK0D5!qs{Tj^CeR~KrZE}Ei9^vK?u_*+FBng< zD5a{b%&i@S!6jokAjsi?H4Of7UyJv~>Y?X<(Wd>=U;gyBKmXbw9z6e|q`QUEb$&B_8TMtMi&@&4$?B-T=0O-dfw+rj9{g|Sap+TW zakjbVf(OeN`^L95>au94R=%_p5wL129`#MPwhJ3f&Te?&?9#wHo2mzNIDw0c9m3y< zcq6`rL2L+?3;S%AjT(oMNve8w5TQms?AY+WO6HoEmqBc3%I$nF;K2i?!Y+(QLwv2R z^uo_G{mjeV@@uZvyr1*9m9y;nB}>X_PPUG3aRF7FTu6}3p~Br|*VfO%icU_;%uKQMUrd>3ec zH6W`QZ0ivurF_h`6eqIfv^iJ@nbObrx|nkLpT70Ke!w-{Ec}$? z`j#o7=c*%rA({q1@b0jS|1IE_4|uxJc#v%w6LC~k#niqHZ5m}qh7B$AOW=CIuPpde zy+XVP#FU-`(L@-2!*zu4Zi>79ILkG9y*B-Dq@{>6)vCN{@|Le-Q%V$dQC+x>o|)*Q z+wdTGfZ6H+bAk4E(>9N?yV16MpR=WotLiv-RI~WVph##0jZC`*MOR0cOdk&~JgDH2 zX`2W3c~Gwm!+m|K>8<|JwjIlAHn(*@yv{!L#y6=WNrD)a^RjQw6merf82U0OAma9>jDM~mK z>?wO7V9|jq9rM1vyHBWr-rIE9&fRNA)nsGemLnv0Ku*Av}LI0Hfu5_bko%(6Q8#b-+Z)PP@I}ZBxI_Sw8+b57HJkWiCng5*~KK-)Lsra zxmZ%@dQ|`MAFzuK0A1n!)zhuJt}_7q?x)-GG4Fc1RFx{@Rn*xL?_Jc$eUcMPrC3YS zUqbl8i|B~?@L|OLTWuF%F5vcF_t?7gyO+bnJljF4Lz1`Ek#4kafQVA%MJ@MrO1hX2 zZF~hAjnjQDgCKZdEy76PWG(V{z`uCGee`ax)-2Bx*gU%@B^w`7WC8p9E!N!~*EEGqCy25sPP;sn*tOPMR(|rbf)m zI>4T({Fr4i%nkCayZFdn7>bSE-4?4BXPMkzG zkaER7YO_iS%EnG+(MHBxj+mRGLVc-TwI$#O>{C~Ow?KcxMD|V4`AcU2rhg#+=HAS< zvro47$=107FVLl+|$_R^>C z2X75;KIgV4ZoFWmOy&T1K!?9HDyImrDsb62X`;3qx1yTpVd}_Cv{Aoj9uM71NC|V* zku(3>Gyp~jv2o6s<#uhHSDugSjzX@Y6Is-aqu^UN%AL!?B(Px^nn8^E#wDNXMcd~^f@4D@RI*%)wVQ>M`y~P*G*yHEj6C4) z(-g07JNw z>W#lfo80ZcdVZW`uG56PndW5bvj*Zi-+DQYOQp)R?J{K!V|4X)jp|?;fhW+WF3|oi z$LUmxB61sVxaAJ*$nqwwj#|zj)?t|jvdwZG+O8=C4s&6?;hveZ8`6+`iuyZJQ3MyL z^Op=a5`Ii(G*+hE_KLhFO>%TCUD`2^^Ks_=v>jctEB#mG;_(x*;9{JQpH0}`jqB}jIWEydk?tf?RY~a4qH8K8SP;a4UAD{?0D&u1hEf8q<1*ZY!ZJ;;~uFB+H|s znETFPxp8tadeHSUWAJGV$~!BwH=McNywkmZEd0*rUxeS?ZLxm-G}rLElZ@vm{H|Bq zQtN-oq3p73Ji4h-Dp?Cj3mo!oFh{b)r~-6}tN+h`fILH~yYI`FGGqO{{d{8co6EuS zIz8uoCAXr*vLDt)Eqv^!e9Z_f(3N>Tm>gJ6&3;Z}%MPt0uKkDEojdBXjd>^KVC zzx&3wTm#It%9b)`jhM=Ht9Tl#y#m=}#ahMDn3w^$)Fj7+gfM&{HivP~LBZ$KZ>SUJ zPmYA$zjpC`%jaAyLQZi_vqY@p%9Dvlshs0(i{>;Is=N^64Ad!5h94OKy}L+vVF}~I zixYlwICO<^_pNT3Cv(PI3p^IgX_>FANJ=)R$kHa82i*d!S4Y;1y;m4e0`Oto79jg) z{^e7y`?!$m&;74o>e~7IrKevKbabneZ*=EBBn}+!wRJl`(t+^2!~*hSEK)WkJ0PYx zY&G46HtHCeH*;{O|GXF7IZ%xudmypeb6_mt6MW5f_wfGHPAp3t^8F~HvP^2(Ltp#g9qMp;%X7|chFx6nh223O&q^^*lmMf)kBjPTP>rM z=L|=B{DBtTcd?Tp{Po+8#J_zjL#&Hiy0VDklvHt9 zl>==Txiv{L(KJ~LV39GA39gw6=VC(FWA5MfH}*1b0RB43nB1_EG2ym+yy>x|8_AO` zg=LOq$)jXaWq#CiB|BPP48a2Q%uqbtqml=bjF}4ttWUjfd|KJfnYQQ0S@w`o2c`L( zc1esWo5z^%z?Ym>$d@CnXH&Tb#^ELu8S_K;Xp_(pWy+`cJIdtX^GLj}9&+14)oxnT z_+Xh#M_!~u-NypPSYW=7iAsO5^!s&(?wjl>k5l6PEbbgX zxtG%0xl@*p_tGb+P$E5&wkS#3?n|Ghs;E*FM6D^9+f!ZyKc!WE}+??*8F^69{ z!cl_Zy~9@9*@hWpd(|P%Dnr`l*^14iTeCH-$-r3AUOF>|m)J8J1VQ*fY6?XM1snMA z{VI*>BJ2KD8ra^h&}pdnST_^h$_S=UsGyYbJU2wFNg;YLMsJju-8qCzA9rx?I`D>% z^Sj7@r&glNH=}+Y}40>`##{^h+AgDC3z>**z%PM4tb>LG!ott(RItV9%*_( zoS3!~5Pq6X{wCtCo^t%YhSTqgjT+AKeud=oGWKN;)pA5??dBMDF~3K32Kg|J;=+J5 zjVA)meu^Aj=$_S1dFT}Fm#3Tro&|?z+qfrP8PDx<^zll!xsY9yPhv{t+HA|dz+}>)2$(JU zNrod&oN^b{Tj0OWrq{1I+`LnRz;bhzw&T8*U9og^Iz!crX+z17Cow-t)tNiB0O5-n zEQXXl;2B1VtLOYxI=%1tXU%>$=9Wh><)DTw+Oq?Xp`fLTtEFN?x*SDS6_}~Z+}#wg z_#OpHeE$I-NN_=bd)c7*0in0UZMi>_Q$A3ZM>5V!1}baC*okdjDebdl78fZ44dFtI z@#x&E5*HzAgdK%~>I1tczt4Gr2I{!b^}f z4a1LvINq=t;lzI!@RMxm-5$PT8}ULMIn658DcWOHiLqJovFMj>lpvZ>T@k;O=fHJw z4(NWG4z8j&i0^;jh@bl-3yU*iTWBtzssWNsWK&cZW7{4yCJl}95+dHM>m_W-!-g z`z)}WmrZRbWg3?S%o*Ev&SHMis$V>*PHU)-aeu2W09X5U0>L-r3e?3vq5l2DTK%z> zOTuK2oATIem8rBD^@EX3RDhgVe~x$zGjRQhArf+)69`FVC9PMTZ{WiY`Jp6pN z^A+-zSMRk*QMGE@XNv4lu*1^L<=Xa3)wB7?OyIAKX&#&=lut}h5!xE<&%tJaG-5lPHo=J{;xk>R&&0|l>g(P|K{)Q1oI&p^JA<_B5;yxoJ82xOR@}8CEAp?jj4zM z|1!`8(6x0)-EZf3;A+Mfe5rVYAM-D`Rq;jw+Z|k6#6!7cHLY7VwoARVB^eU6mr-3* z)C}|quZubAGxp*3{jF&aXWSLs{I!83cz2r#lECsApHH+V#CNJ4nB+rR)nV+ zt1}}IAYj3TKdBM_loHhS@$0b>R_H(&VWa zhxp!73>t-#H626BC0B-YtyBKG^{9h#>9GAO{SOPo1Fcx2>CzHi4Pt$ zz_V3Ve1E#;I>L7}?dv?^J#7Y5CxcipRC}AkBE+EhVVf_~%O*NO@!4YcRPh5Zyn%=Sn(wMht{ zfZLNKB(O(tGgF=5d*OCmk)f2h^zh%XMB!D*=Mzu{({>WAM(9yluh@ZbT=?simmjCj#yZ6%M1pV9+U zy&pO1vohZp&X0LdvCV&onH}N4%v%ry{v+J{mmVTKc-&JrnGC!6M0f4aA?pO?bkwoZ#a67gfp&Ohuwp1d`B_I&Tik&bKGqNeQCB!9Ucnx?jRZd})K9Oh5MdXEI&nN_ zA*mXLm|-f*oQ_E{<>)BVWofp3pwlMow!1w@#is@d`oMk?`s)Jkui#FfJ87OD?*6Sb zf*rSF>BvD)0%O|cz}Iom6XoLq$C@t3l+Ksi-EO?cVHAI)QpY~-c6b`?7tCC6EG!0JP+sAk7L&eK*cR!UF9iz-RXV*I9?=1W&Riabyr34Q35OiXoi zd;R7($VJ`MGNa}OuVuMIQI2@4DdCW%o$S-hZouOx_--XBis>i3lK*-O{w0M0@<>AN zEZPn3@x3$|;#z4+7LCND7|B}mbzjr9Bt+CT$g$Y@CT*I`*G=>DjtZDO{sT*=l=&*z z>0{H}m;@-e8*$gB*zMBw2d84$5zZ#O4|q+B)WmpGt@+m3h}&H`LgEO)2h@M*U#B&{ zia2>_rwITE-Ho{8KJ3c%GEYsz$5J-yGO8+RSLlzUm5A##u@NVsy`BWX-i+~=a(2pm z%vosknCfb-nCNb_9dDZQY!F)PJ*RD-sI-<^sFQ&WSt2`o}7;!XsA_I0R!-mL(_?vmG<2HMuu{aWbthl#7c zg;lOpH%!>0HQP-SBE*6y`apRec<@c85A+0nWO3oWglFnFYgctq(sG@XByG({ATC|k?n6EP`S#QtGzo7kP4LOYAT~F-Sgkz`&nV>(>zBFX0dF^g) z$hbCT>U%vUNlwwS?mXFstjp1uc9lt&raZlPaD}1m!Vs9y#3A}fLxy<6`ca#mL?pVu z+v*tftQdO5`w7&{*K}mZG&Supa<$Z~lGIAz47}*Nv_#&_-Ctus#lBkXkDa(qwhp>m zS;Dz9Q&ZrT;X&zQO811eW0MWpB$rIeCf)MIzQixe5Onkh|RqVc71m@dL;1g3*9|;?I z=%R^nKyFUY9XHb@+EU3Il`0S!On5wsmCrff`LLV(+$zB%lZ@Lcf0RZ6ubZqs>_=XZt}OE*PwM?x29_Rjo-VwYYdKEOU=RIw z-zf7S3T7Vir1N%a2E_q&(0OBy*VFQ1-4$8c+V2Haswg2Js*FIobCfl$v^#IqCN+aO z;K>JsgQCCAhI)<0{>6EG6!c+;u}S7!tgN`EKh8Bv4$-Y>O>|yl70=Ujs%xF#Y(WjT z>CMz$S%D}zjaB{U?il%)pMLMC5+aNd-0P^~8tCMxdL-J8Emgf~Cf4+zX@imykFsor zOCA@Hs|J6>2zzq_`K1nfm6kR9LLYnT=I-)@+wx(-c zCemg-n%ePFoS*CVCflXd^w(C>z#rTf0l+hBkAj;`Ra}G3)<&QvpHWlgP;lNV`D3}WIB4yXOgY+ zS`>UA8+*Mx2roiW=8%2Y_VlH!E%If1YO2$xhPO0y1-|1VoiWrafs>C;#zj{`p~DxA#`PCe1O82o+nR6Ee?~?R}GXvp9;vp1#9Hbt()a zf^ZyuKmuI`y@b*WOwByH0rStm|fRi!;{SF!@@Gm2zx zPh#UZxG-#j0ELu&pt|=_?lJa=6dzp3HSZqijstu{)6l7uuG2KOMOxr$@9H#}#l(S1 zd9l!B^dcZ=?rnzO)DnA*7a#iy@c6MbK^Wiey2czA_*pFpO}cd4x0UdHWd+ZRaV=1u zj+iAE&lW;$qx&F&uX(&*{v7sUsmS8siPB6M+-$nWV#h?PiYql%WLoO1!Kvtzc3%+N zj;_*0Ic39zJI8EMe?;-Sj!dttvPC}h-C2wt2REU2>yCVR4Ldy0d!%xeu&xsohsES- zg=C$^a+0Lvxs{@u2Rw+s~D zEGlL`&9y;i9__0&w?fs**5vVcKUmjb$)|DPWJ(iNc34}QV==Y0BmroQKSIOsW#$xx z_~EG5oLoR|PVUX~9B22k*^5Ii%fiu?V%nbh67QTuqfjONdtC_(T6`A`~p z+IWl}x~2mJ;oUiaE`*nMNj z)3)QEfPd4m2}pQr$Hq0>S;d(|dkT%pT$L^HYm7GTQH_y@&%BPhQqGmuMv*V zGV4vcl_?h4|M>nuH@|bn_&?_2CEv8eRp5fBLfx%C=Nf7tod@|Ge_FP107?y-GTAT1kGXBWgbPBms8hP zjGDu}O>`pmrO`cI*Fhf_{h9cpMsl|8KA}buydXOC66kS=o@S$VCmkH^cvX^Z$CQ=iX6r9wf&7zGjj2Tm-m>~QBecD{=?vMBweMg=Axg`(5&GJL$ zkv|^dScE8>`ku}p1n4o7YRdV15Ck)CXHp5pwix5_G@GgX@HD>9a1M)HtRUnmL9(=P0S9mU(fktFmV*(RC%toU;Z#_&`EOJ+xfE zmSQgRl-SgcCO&S}gLb_L)5vnc4OF;bdn*gB@k+9Zq7E~VDOmmWD6&HwVn$-Z{I4>w z*QCZ?o1n$)v3w)|+>5wl4lHk$jbW}t&WS|znv1PWr@ULMl*u-+5f70yF*QfBNmf1%nRo+8Brm9)@ywUayjWA0A zJf@7%;7!fYmuj5)ZFtmK1 z$7-H!SGMrFvlfE6`rmbUc%o2( znah&xL#@g*@0(SP(qx51jm1oMx-vlf#d@1vd?JD{ViEd4_79$fBgkyLM{UoOs894TfP2>!DXIju4>HU4-^iFU=M(Ys%dvB(yiE5n_+$SdoNianA|=_{y96y!^*4y;PE~@!|&_R+kqZB zE*1B+!ev47Z5cXDRx+=h8jn5Q%#DtlPFxuE0CvkCF@!(hCxb&jJ;C#BegHWQ2~dP@ zhTJv7dLyUE#d35)j=JVUI_$H?Q`cABigMjv%tb83aTG>0_<$Zg2JjHz-;eXobr~Fl z!OeM_YxVhj-qe(BR4h9m=e(|pOlQm32so0o+iV6VyGYf>A*Elo-hNMWQ_^|ahx-TM z*ZB_6@J?G1=V+6CNoBd^(2HSRdq6FaElJ5*6uM6{tL{@?N_RzB9&J7}Q$r|ceMJu{^aNv;E+&Js&U1z?6L(G zU=QIFpfgXPN53h@v5P?-3!H1*kVPyQZRsYu>>!oUnhmK**CpDjWw)zweA#|AsTYmt z2L$R6^)!SC)iWKUzBcsQHQ*?38)_q)s*ukEu9;KuSHT-ip%PlGM%qWZ=h02;;ql~-^Co-TQKxU_4o zxA10drLPW+%UN$e_Ia1Lg?)WyY8*QhA*}7?P5!(r4dY-W#(7Oc| z_#6M3TH)!=$JzNnE-rNHnxt7IBHwg<)Lqv=w@$nnvV0rMUhYdV=eG9BbxeSw9}*;= z;O~1F28J|zHX|_aW*j=n5oi~ z`LEs(@lmROZ&G!NCKAH#MyL;*pU!>J>F45N?i;$E`OkfqvpGR^!&}Nqjk^3`NJBcddSaEb&vbNY!N-8k zX50tPbnJnK9Y*0(D`JA#`|O_YfR7ws=6gB!1u|)TG8bAbYVRxPwzlOmNsDc|tnOX~ zaU&LnU)Fhk>A8d=AHbt$>ASggAPjHU);b^S+JZW%*Q1`hxt^y?SLM*QogSB2j;JH=j5$4^mb1i3-ov0q8wH~G+xl}P8VT!)2rL%U{`WKgeD1=cZ7 zE)KI`4uCj}9su#9}<0Bm{B%Ow|@5gG$n`|wR94KaeJ$5=n zWqN71U&z^TN}$Id*qQaDt#zi2PxALuN+HJ4{pY%Qtm9B$GkG8kdyAT`7}!`YyqV^i zt3@lm+d9Y>tbkBj13RQDIql^-G!$B4zLZ+> zrYw(Mc4?h2Lp=4882gc@rBX4Tw48xNy9JwQ>J{NEfd6kmg_K}hl)3V2?}q=+;J<8 zs5V0j8HT&P>#i!wELC$k=knqrYx^36hyXU9GWf()E%bE2KNh|@*Q)MaZ@`Xs zXuEbP$ZCr6T=l$$?3h|HDk(`W=^<-o1CA~Udwfon0_J~lnqp5YY?#sKHbr6h{)*!_ z-5AUqgI^neHVR~sa1+h=Vek)L%YB{%@nO|DXX%->@dN_UpB;XOmX;uR1Pyw zH~VAM&xJ(_^zKQzYtU!0?sDZdFSS**-T&W_q#?t9g!9m)-5=@s=$@{VFe5e@tY zdk}dVwZNlwmLFzSVR&~>4qR_J$hK;=m@&=EPS>PS>yBte3eLs^Z&yj;?<7Y;7Sj*N z%@gclgbv?JPn<^(iEwbMW~u9593>@hnx<=;DM?_M(_Ye@93?DklX1*1;#BcfrZ&Qa z_&?Wa?6K|s0P=G^8Q;Kt^^1A46o;0B*v5fPTRssrFKL5Uv>s|`P)EEX9kGleWq?Dqav_nX5s=sz)*tO$o zum*V3VBj_X5b&_V0`a3oWb@YD%ZdWW&GjOisv|R%RY5yBw`;b`wV0+I=lPVB!)oW~ z(Pig6KJQfgf%TP$de}K951Vf0gb4|`Uu)K}giu{=D-pZivZd_NoQ$eShqYWQrT1f1 zs~z0*!g@4^JD6hqij5uoQkcJAhNz`li?5~ zmF*_byv~+zSo}3B{e|sp-VK63ARI>iy5rw#qG2R_<}qM?`*zSBS7{UEx;B}+Oc6(w z-litsCC=()-G-xCh|Q0J3+NEXl)byq@RG%VkDVm~k4X}Du;FaH9ZyyA;^26m${FUd z8EV?EGd-lp7hNswa`4Ll^^&}sZ$8Ce^`!5qxd%f4LW1W8G6*^GbT`j+OpifpneD?q zQH3FIcfC*{x$e5W&%fUw*3eu--< zz@@_LAV@isAheQOI`>W03pHd-IwglSQDKqo;v1Sj5fEk{=(MxQ(@F!iZGL|3r~G8H zg96*(ConZeNN*=i(Qh1nP7xGO-aaYh2~H=`v=yo|&Lui_S>#G%mFo^!tLqW9g+nnR zQAj_L4ET_fe7FIF0{r%!@ax3gaec>;NV^HTq!?F7Ql^fnkUc?Y+;xfyyLW4@>-eHW z6321qA87EUQ1I=<^^AK{=%8CCGp=o)b(wBts1ur2po>nm1y7f(T4*~TbAH%Zhv9`i zWdH+0KY_N7KK&m^|KI=opWlcRoK@$$VE^kxq<4 zcUfkPz~<6mYEo|Ozjec651HPV%=?8a7YC6^apQYWao5uBy;#Jd2bkM*D8$m!Z5SmK zMCt~#SnR${VE5@DfSA4M*YZ+Q&wQ|E=tG6b5TSDnb#qcrT}wm`ZB|v8s3Z(i-B%d0x#2 zsCh?oI!RIGO|x9^ww0s+P=pBc�hY?rj`Cc5*khE4@3svE$=>O{;ZE7}9G=bS^un zm|o|yF8ey!+I+ONC!&k?#Hox((FbfBUTDU?062Whf}|nE@V-aKuQ{6IiOUM45rnIG zSvp3mhRz!2Jl!dtHP^zH3=6*~k3aWk5REW}4{W9ep(jltO3+gU0u0gnS|i`|VySC0 zF2S0XiKIW4BPBgwwz6SEmSs!AN2)4mFQSK+sS6EA6fo?6rp_4g1%C99NP$dJfp5Oc zT~pl0Ny~(0djrdWxhOPiiasfy~8LiYd)|N{Pz5bf#5j^;Idm7&4t3Yy=Sfz|iflOZmST z@GO2jf-x5$yE)*Q5XUlsw`vO;pRIZ zMltcff)0Tvq7)0FCt9E}zMBTi0>?Q!U8ZiQ-IU0=tEUAwb24-wQauiGB=(9+9AAAj zGWxI(HyE>UavUK`zFZqX(m;kpqN1Xrsn5M_co3_Y%j6sxol93*xRo6-(_o&<1D?env|+B z#rH1ccMLP1t{IhLXnpjynx!S6QaPB5Y{-Dg;F z-vK8>Q`fGU)THX;R?p3_ioIP`JK|keW4*L{y=IvuH$lT7xWxN8TL?ao%(2)59V){3 zX%psb`a5t-jj(H(kzphU#Td4xuErgU6=Nx2Uf48dc(e*Og3)ER5{5A%{*$>N@W5Y> z$nOFk-=H$`eZV{J%7|&9bFWxgC+S4@T$HRj#+e@p-Pd`x1OAJ$^GmxuM3`doiGcTz z(Z!DoH}e)^bJE{0C>6VQMpj)4-H!P<>1Az_MX0RPV*%5ibV^<8^^4L7#WBM12RN^$ zp77J2i3594^&dTzpVmaf$l6gg?*w5ezO#K32jA?cng{#ZPo>*j_1TcFi5Ti~q}#CV zawDej-$7Ao34xmm)ZTY7amxNbc~@R|Pr~0_4J5*Da>3X!HNHIZ4o`WpDyS>Mrh(vH zY$`kUBU^j0teYri*HYl5(MEy)t06)jT0CF1{#78@V2iSQco!Wdg94s3v+_ zSnAh=-ErGak0e`XmFiURq0Z-Vt?0xi6W29ryTVeTh>`85jH1Zon4$l`4t))2;>YT8 zp-Cq3-IKZ4H5pzHVlLHrbThVRm}h;RBhX#l?ER4avB~uFX@2qbI45(62F!ms9wHA- z`Z$7zfRoc(nN5|Q(>VG^j|{a z-pD0-nxx0(yrFM1Y`)9U9TRMLGPUWv>?@Uc(6xe!Xf?C8XKS~rwYMj6wsM$B+2gN8 zX1@d*Oo=DVzp?HK?rpj|##o5HWoaviWZ#!!OHtFdYuA&ZB57$*zSy_~7xQFumSgO{ z`9hc<#$@!&MmIE>OL#Y5(lO+E8B39@`L;tV3N?;9llwa589}Y4g)R4PzD9ace?<7* zW%O4XtmuVh^#Ap@|NQHa2M-9*rUJcJ{V{UPp{jV3w0eS-rMkphSF)6iQ(m=GQ_8)y zyXgS&Ws@ml7=q{n*0~;7l99;=AMY&U2yWJhJD=xx4SgucK)bXWR7G=M$`Tcd&ZO65 zSqM$-x;u)ltf|eyV0MH>IkfT2nl) zV@m4Y9-rWgcj_sA6NMj$aKMqD8$1B|L@F*O=x)$m+o+0NY`E=!YA&;|af5WqatW5H9F+z+}`95@-9d~Fmt9G4zUVV3m`$Nij(edkS4 zq=@bFgf_HTrMUT8F9$qE7~bAeeFb_NFdxuAiFFKcd#n@3u|Coi6@1?Z&FZW`xMVZ# z2SO`diK$YDO^6`2T3m4QYk8pw zQu{XUNl~F?X`73`CcxEG^=yie4{WQxpx*rw7vj`Ce8PJtVT_|7x}Ep_r`)2;B}Ohw z(#oCUiby^R-M1qeMVLb*q?BHK77eWU`645jA2>kX^`?v1 zhP=k>u8#qFy!D-s?O4mSoO!d$y|-m2s7-4{)SS*ASY)KsM=lq|zj5TEZ@1vSMcpy= ziDqe@92HWPN=)sxf>b(@ayONnPO8OLfi=-)p?(UdQThqg-K%>?4{RDD6qq9_yp{YC zIi^BvHSOeqjzts=$@7E@JS=z#Dk?~6h77yGFM6v z-i^28dYl%lYkRV`TP^4$C7=ek8-vRrj4L3Q6Ia#*_oVDF$S#-o^ugcg^qjb7cr#CCHzo=iUJ0e&qcvVF_ow*tP4H{biLpaGjHc7=#HU> z4y^fx{U+XTkRrnLcO+O>=DPM7wRM?RdY=S{MTq(GN%t zfhQIJvqF1*)1S$v6Yw2Jz06lpmt%P}+nOZvmMVaTn(#-PP{X>};KN0l0HT0|Zz++y z3_kM({zI3Zlh5(KB88RPBiG8$3+{n8v{A<}9QCZGDJux*HOV`fcbQFe26`bn2WJtB z;!l|6dT7R?r@?+o4uvk#dW2j{jg<`_IWCX=SdTv6pv2WgRgsJ&&9%4)KcmZxm7QxP zvHw`d5%B?g_;~5u9P!b;9Xf;@w`7Z~(8E-$gODk3DS1xn6?5IQ0%dw?FK%46#uQ+N z8GL~B3?BNi2tQs%Btd+$0(69&8*3J0a{%S}MtTZ!yX?!f^yL8~Ti2Y|c7x9Lali=T zF#f`+lDk9Qb|TB?MXVM8`<~dAQ#Vja^=j# zQE+z6J|GE<<5w_$ZK968RsKHB00||*t=t*c-VKL=d@1u@Y6z7wH>yJc3yjtyY+FIQqRJD6c@u9OQ`g zwCJ@+FOI4g@CiD5#~(Q3UvNhM@BjXcszO|1QUAqIq%w(OJqpuYHBuxTQuyn zP71Yjds|eP6w`1S#a!eA(I2yj4~*V0_;Gjm6Z40pDH4H`<{Wb^9oaWjPNY6+U3W(h zNztQn=(l~F8$F~Ky4hp6WpjFhbFztKAeY_tIjKr_XXhRdOFtxu zEEPTKl4LYbc}|3!=7MA$$*bBv=gC!;fQ1YY_RpxGv8SbF{3QwaIOq|k4uambG9YbX z?2jfrdh-`;GL;W`iiho(vTQnv*`BLkMBLAmF8%<2&iu!|kbf{$|6vSDZ`UZm;JAR3 zYmIUtrnxnRTc6Z`$`Wspg|1NeZNRKz?{W-uYok9B-uM&EL* z6UWGJ$<8H3X)^R^oU58-hs;`x7v6V659C6ju-8|S$vR^Az_CsNwl_)0JTc`SZaI;F z;OKU)l{hSr~zVDr_`%2+5aASzkH=Vj(QhvqE8|3(R zg11BDSwAgFvf@!@>7Y$6S68~2sU7X)0=`M8gReQ9-*}G#A2)g& zJ*PyUa#K!#cbv+>q?t-4R(DFQ3qB;*EhcHVFg5K}GTD<>@=ZgQGW^yR|0?jI2ZAKn zUR}R#EBxe}Mc5x^TJX(%ETUWN#Ob6P5f{G3c z>d9sQ`Fc~FlT-S(l5EKS$M?m)`JJ=D_m7{V;Kxr-#Uc-rItZfQkZJ+BU}b=7GBxjU z-emGnO-1;;;ANdiU5KKQU9l^H?e<=HVPO$UA^OBwL1IrC0))?Y<-!n~SJ4Uhq2u%r zQ$31fuJJk_`HXluCpCv;sbFrv<27u zauVk~&FXeC|IhS-XHiUOWV_yqMorpk0>trUM>e1_Wj%XE<;Z^NtI` zu9Fyu9lq?v2be_Xx8M^oAA5uT5eqLeF;88Dykpqe z+U8v80huI!K|7gIiDXk0TSk*KZQF}<8M{vILj*YR&%y)nanxh@%~2l&Zg6+JVlmWc zChOXWW0I;&Ye89GZ)Lv>Ra)g0vnd-8d^5p4w+udT)JNb0c=8;0Y*KJ1z&q~qA9Olo z2@*DqXH(UQs?7P+FHG@#=%>zZ0A23*5Th8=Hyt})1m44{Bs^3*3S$zXyJxq~mB#ZU z%_L?`HZR>UCyNkuSJb*C(Bpk+Y)M*%F02Qkt5EnBY z*RZp>DHh1ZumNd7t7OhlT~(=`#s+|1bNKjbzz)&ZZk6ANKU=%bVJE?FRHp+79KcQ- ziyIv?-$+)l1mPVYLqai6L?&Ub2^pX$?L_XXHE%9u;cagHgrGO4n;(5+zg)T#ZL#<7r#-W9D&Ew_t3P;ikJXN*z*#2iLrKVvrn4;Ys~ zg7<|QK)$-i`cn`6Zj z-1O-xg^6Uqq40A&boHu&3-V1#k!Yh@5`K|(=pm!+)?n&${b}e{zSB6 zU+_o2Q6r857c@LVBiFGn*gEx9w@>quP6=rMK?!}+<5i9}>6V0-_Ip4h7RB%Z+HC~L z7x?khvW?*AZFBB#v3D&-jK`zIoXLricAC|s8c39A)^=QVtLzSZ$gFA|$EKcs(^KNL zQ$O^DJbSV(`$P7F*{R(J&WFcn2Ztlx|X#_f?QPQ61B&F*HQ;CEU3^iPsUs=0_u*tTOF)7W( z=@pquli}bK+clvdu^YmktKFOy-jS1#cU;nC8x~2P7tpU<$$887wx&$+jiN~jm~Ad4 zyK*T{3T+hn@76@j(|J!HO3(igAt&hFb=fPm+HuUwM3A-ETUzBSFWROouw0g@XmdTK zQ$}s-G!Crd&SYjdMxp=o&fEa*`!+OqK8p!VUE+vOMICo%;W!$UGdjsyY-Lt;HHK}& zl&h-iHScU&Z+fZfO$N&FyZnk*Br#YN`GEe>j&%HEg%aJ{vZKzq@>;8X;9L*$w(CXY ze5mI=W7%==CB>G;RUF$Z(%58UKs~z4e06Q z-0Wl@^w#Nh-Hy6j#D&iM;4 zZkX@{i=irXHm@6%YTugzAGAZLXwZc%eWA%mA!8B!KnefK3-BWN@OWa zn6uE)FxK6nBjJrK6TnBjXTu+(wK zrb+SKh-08Mo2Ly_yDX{B@?8!Zx(&~l3G6o>ku3PC>ikk`0w2S6`eiKgjRJiL=N{nQ zx4Uack#2xm^sL(llvE059V2Mqui3CbQYLz+1zz9sRZmrS@U%m>mK%5KvvnHU-zG4Fc=W>3DDuc0O%;X+g!vs=2MyF z=6bbMz@E#zc7|#)-$O zzklHJfByB~{^MVMLOrB6(CYs2r++>2$3q-5Y{sM{Bh7NYb@f)&s%$8dyQXQ<)hW4n zhh3#BV{^q(9Qy~~e24si&uq=3AHV%?|MB;Kz6n1|58?O^J)-bu9W+YbP6K{<>K~u$ zkLOqq|KkJsQa|cYn`feG7Z5Y9kFsmQl!N=$t!qb0jxQcH7o(CeI{A5TWecyW;4lwD zD32|P=-KGG_o{F_r9V_L?S~e9Hzb&C{R%2Sj=fkH+7`XAIT~RUSeXH^2))a*f60yr z5cUQ2KaTOAS=ab|`Ga4t2ggVQUkT3=hH?sh+R1gy8_^G_o{O$zsJK`R;)|#mg@8u! z2efWY?cuKoVD$Nn-qe32c-Oq(+Y!BMLKdHUQO$IzkG0B(7!+D2+zY)?HEmf+Y;PL7 z+k+!yDwPlDTY)b^4vU^*e-bnT;)b8W9G3)XE=haNYuYT&Io`9mPHIN#6lcXs!Lk^Q z*wv}Vl!YcQ#{O5TA|LW*(IpZ5X|Xw5GNJzSQhO|?#<5d3=+MLy&ZBJy0>3ZW>#_wQ3ht_>{uvQB{)RVK1y z!Lmh#P?Id`B}GNv)YcyDxe#w#mz;{NAMhA8@cTeLdZ=r4a`2*i`!wcpj>&hLw;O{r z1Cil*Q+duNQI9aM>5|iCvia2{vaRO`X7Bn&zY^nN=s^%6iXP4A5ylsy*#WGE=1JM;cygDirEL+bI21hBYs$; zQ&VcjIJ&z=cMSOei)fY1a;%q1HSMBw-lb(L#-T{}w7T?U5w=&NgD`lLa`#&I9eaiR zQJs$C(1HDkYt*w@YFX>P%x5)@YbGd3wb+_u-x@-*i$Y~|q2t66AR0vY?u* zhw)>2Z5Y|hGH(aFYa5vnw?|s<-M-N|ov_OHdfrDa7eT9Z7K`l^Z7bZ6m{NGR5BEw{ z9uQA$*chWHX*(k~SLQKx+wh$k_>@o&uc0a~d^{iHnPzI>!D;aAk71^3Zv9s8&@%>>UN5O0jBF?ZqC>7O+An zA_4KAwId%RmY9+!k&f<<*@$sSQ3fX9zVDu}+{in##pI9IRry6Aeq^Qe(*N4#$3o&v#5Lk5M>aRK~d24yNU zQ>4V70Nh7y#vYATOvV{tLhc6KH3=@M_o>=Nzp#FxaEyBGjos@tGbxJ!G#2O z3VhQKP}@=HNry0wpUl$Cr@MQ}Id<&;UUqp6>!2u4J2I9;ro)!DYD>~}%a3-jb!*5) zwyO&d4Pe~EVbiH;NFHlqN&0Zz5 zNcCJ(K{%Z_x^5y~1cT32#5n%Q#)6;D>KE$YvLQ?&fn$Pik8;PbCo)T$Qma!|P!xm` z-NRIl?b4;VtX5$)*@%6M0~pXgv893nUzi8aT>MOd7!ns=Am?)(`$3SREA&dBEH_w| zwv$>6vu4mAcx`RdCx}^im4~N5*$0YofY8GcACkw;ynyk|My==b9IK6GbW`IY=XF9n zse7I7Nu_jBr#suWi}4c%yF5E70T4zXD9?Zeo}wK+PUKDsO!$7Re{;sW7L&wXGp3`M zSTAZ`qnLicuru<&fid6z@S)&XjBf97^#PjAY9D6kN8KKKZpSzaY$`M0F!(^ZMmz|Km~a2%RRxoR+^qh8 zKF%?RxxzAm&V+R`cBDv-?P$|A>lLZd%7%*;yv=FFET%#90S8Bds0Xz1Zx)Szj1*jK zo5q1-_0T-uWexkFQmS@Awx-R;qT19jDOuH-Rj)n3frzTut%nd;IlLF3BsEvyUsYE z_^};x$S}Jbbm}4(yk}Ue5iSrv3$AgA6}R+gaz>BB^(!XX_PE2zLto$F93D<$zUPp}yz~p%fjfI-$^%b3e|IFI?NH`b8Ejz$l7w5czMx zMK2IqU)fCg6YlT2?WYrS4D9YvnrqZ)g$i9S<2vQAFF7T0N>{S+MGs;s>dUAcGh6qM zm~a2Pg3zzTH};aiJ@MisK^W4T({Fh2lDk8d_W&P7l(nEklS|^&c`}V+t)@9tEtl0W?Ra&uefM3&bGc= zHo=rdkAa9@d^j;-Z{}|=t)UTyJfI%N&o@-g+s6Uwu9pw@{%B>pPYusAvCB1QV}!V(7;*V&wbHU zf@aC4I@+hDy>$xLf(@OvRmeY+oyOD$^zg9Tetxicgr0zC|Gr^>jtU|jNkX~hOJ zYqFvxbwM?kX;;pw8&jRB%WAom#3Pe|v+x5Ujy(wk83f|zTp9C&(80Y9j_-B0vx-aJ zOf9K;*qfp&a6cznv{O3E+V<}WNpMl_Mqduheu*;F*Els~&wN)&7~di7`W^HPT+8}4 zbGa_fw4{u|q^Bfrjyf-JNtzwki@kJ=F21W6o2(9hfF;F3A2&N=&w(e%Ndd2K zU8Q?3>m=FcG=Z)<_NLp$WL?J{cx7)UMptj`5Jkb;?B`dwaV!Wtgn9Iv;~jA1Aj%!1 zKD8Yde5mlud7E^V06G)0jsK6kFI$q^$g=#Ec`1&K)iew69S?i};Mg3SV_#lqtYT_P zU6QEgQr-IdaU_#s(h(6UM!*4)GNX)<$wVOI;9kG$Ig7uCK@W5zcaW~diKqkF0YXb; z7tHPRj1tfW*49azX_j_TpfE(kl+=~q_3;cNoW>(4!^$sQ80Wf?gH=3Qx9fn|ShnydxfDCTjb5n0;nD!Be0Uxr-tzp@RA>`h|8iPQZ62BUwa#fPkayAvW zB8QR#WEd@bFH=7!rKdu;r3Gn%0T(+e$8+jOJr)g(!99=hFFhNPUi9J`$J z&Lf&{XPB;6*DUPZs+)O@cgJ|A^c;XESO5{Vh=D*3m~OpfDoaoJP)4N}dtTU1nNXn5 zJWorWM-u5t+iD_KB`*YkTPI?#YO{hJn(F|tv)fEdWoGMM8vus0^aAqGmr2odJ`RZ= zXLUqUKck7~Hhd(RzU%96Pbe0o|58hUL`zou%hz3r*S9=8X*>=2bmnl#dW}b6Q8B z^rM~*LyQwVjak9#Rq0iO+i?J*)N%G7fTs3N3l4x@Abvz(^Z3nhp2m|B6Ds}5ojZ;Q z3>y+95{)W;cL~W`EqtSRRfnT!nV30%_Yqk;tT({3UI-at7k3_9jk`ZkdVVQ>P5=Y;0n`I<2M$1JUl7aNTtCf$h70Yx5IxK4j6c*WG8Y!NWu9Q zDzeem#TfN|hQ}TyqFKnuV7=I&qItj&+dord*U^Cf z-`95^Hbj2m@jncFMKX)Ipk<$jq~Lub-MaRLZtZZz z5NJ_I7nZ;S3UW5v6Y=$YyRA|yz_y)lH@&VnWqk}PF8P>Y0m>;UY1X(XlD^M*rjhrn zOx;@d%pIh1xC7uua96lBP9dyrz=F5tYF|(a8OB<0n7cHcByiov4XB5Phy=Be$&mIv z?^Z$$w$ABI03(M?K7t*#&`-lm78Y-`Ep?P(oFM{UL1j3GIHS#wdt=`Bx#x|3P?it` zE1cJDGzJ}Yma+q=;r3{%Ad8-rwg%AIyvmMDvqqH?6_wycHV>pgM0Qm_Q8Z7n@VsOS zHN8CepwRh+ULIh|wAub6?1b&0!Gi2lw*8Us)5e>+aKqtv-iR{ng1+W4ZmPMDsw9uI z7E3=QX`cwawz^{Tg%%(;c1y3StXgCjoSztN#LPCZW{yk~45XwIW8aU`mu=+bw3W3x zh&FT+Qi39tI%yEqcTN#y7Pj_J!YJEet`S`vXazRI3}FKUt;qC1YmqlRb0-hVc8rS9 z6}dZh0iLU=8Y9rSdhWeq`&^b5RE&DW1QgmMVuRo;8z5TtHMTy-O*c;ct{ZSS26+|@ zMeXNV+rybwEd@m|Q8H=S7_!Lc|XI1G44d?g8DL; zdD1mW7X_W`i-ZqB+*O3E0AGzpsB8%nfQ|qh?+5sqEn~}CcT2Z9%S{76&z)NmUN#X2 zF%Kejs*eJfO$*XFuj*Jo%h75m0^2IXqF{esuq%LH=$x-Cwn2O$m}AR;ghdU6@X#<_ ze^P)n4l-WO^VCt$&iT|q8G-`wN|tD~Wv&mxK|l~D2aLHgw2Rr3%ON}fw+};|gx&D6 zIq}O{P;V%vyqCBtOOlf)A^pUwW*R)I!;V(snTSA$Fg+lI--BELk@Fj<#I}*W^#M-| zhw!r`>U>6S0BRCWb&ImJ8hzg8UQqi`FVGOW_Vf0#c}%SXs_c6W$Kdk<*eboSZXSSJ zZ7~@3?IZ2S8lxr%Mitd@pOU!6S&D=|&GqOeTH5x&aPgSkSLLeE+Q0a!tYIg_?Tgc{ z3)kxgW%rZQuGnxz*3H8TrBRU7a2DBEd9tc&T9k8Hlp!rO@l1N6_P!+5&aJ{r*93Oz z%s21h@ddS1h_)w-!luGTF4%N=t_TLuPVQ9YV=$(CC~6F*ya@W4lCDe=J$kqG zZX2RY-gdz7+a9$27+#j>fXk_FAZf#&GjX9^_GIW zh$`vj+FY}<(pUoQemrge`ipJZd*IA2mJt9vWz&V`|vqId6dLjY*2~azb^} zcwi!d@J6`McZY>8wuCgE0Z0+G06Boq7w)vF^bo^cuy}+lLv0qr;J2RZRb$$Zbv@*D z(DK+LGi@=9Bi~1Hpr>mmSo$5PPF}V@{A>@dPkd?r_+BX4!3Oq^vFZI|nfXav$g1X1 z6MLenk}0odm&d&;azy==(dz=)Pz8K8-M=tnW4f2oEM0*x%Nb)N1R7DBIa zGp59iM;De{mKk?(NwO}T>4=daE3&Zl%sV0oQ+fbpyS>-KIXTPwxlsEO8*I+|X*gvM zdDzBoEXLgLIyj~|m3@{cQBcnz2d=NLol~9Kms0g4K(fD)BXm{vwhY8xf%_%NC1fK& zOw7U(!=>1&a%Vm?r9S~4Q8X2tlNms1xY!$hyLkME$fXLmlc4C7h z_-sSA7=U*0~$${nQi97Hbg(K0s0pgB`}* z3AQgOaaeeBybWh*18(A}#fi*QubxPikZG0@NVTT&~D1tsw`PsmPgbzywX$x>GimGN#Q$>aop1+}1Sd%A|@y?ObQrUTJMD zMD7+npz(GEGvL+GuV!GE0=#uQ0qB=Qu%^Kl2*|pYU`PS)+@PAE2n$q34R#w?cZtqB zves#^9TmnM!RgHK9w)kUjwDRohO=cioj9f$0Fz+$l46V~AtE8E#7UA*-BbjH&ttBe zwXSck+%W*qKA;-27vUpu@a6s*W+wV;#P}Lh6^u!Q@qh|9?dm*CMt7J7PaWz)mUFt& z&%-NLXN;T!x~c&0P;F?l_-w5RFNx zYoh{ZrH5Gyy$X7&SaGj?sIN$+vuZ%c5fp4?(#ki$vux&q;+NSfgoO&U5Cqhe+ z#zD`=&P(A`w_)9psSb5`KnL9dJB&wCa*3{=Y(Do)O%o6_0gN%0`BZp* z>LzZ{PmK&f-;wqlgKB1ip-PzUauL!LsCmh1mpCcl2Rim%mMtb?J7)a4;gVdG ziHwY6Pm>Hq9%-vAm`1J+_-VvJmm$rsGixoVhoEC4cmM?fb9RINlpu$SCDF!)ZWwGd zpn909g-6KG<51Lc0{ujgC#o^=`aw5zx>jIOAG$?s^i7=NSLSFAxIiic;MM~Z)G-{j z7b;MzgDT~H6wxg4Bo?Ul$p}EogSn`*s2#6~V3o;X<{UsO0}$8?^e5QeAvVJEO-&1S zfN1cu+C^y+1Uc?RnflO+RJP`qSfu4@#(iCtaU4S65z`ak>v_Jf1D~BOze51*ZC$+K zP>v*|MxEbOAsH$@1?d2V*ZFPTwq+r@UKhQG>;&GJGH}TFE8P?CQzOp{-Gps9n~yvj zMjem58j8$Mx}dMPi(I!)!&)V3ydgXYT|J0{tDXx}W@lRmbXL*6Mt)KZU4skxlFzQp;hzeq_9x4)*BpOl=z_P2xuy7L+WGnSd2k2|cfIz_Zw{&3dKq1AzUMl=gVL*OkknI1i+Ku33Hu{*OHEIIAHu((5& zh9UNZLX*0612O!A?zF*~in^ymlC`o7KqGugIqu34WL4_rMG|W@8VpuKnigaW#rpvw zO*Y=^c*p3%*gHmB7vR0-1FzSrcd|UAX91ktwM`e zfzEc~jbp^RVDb0znwAM_e?~0RYixBirXK9GC~Z6#4r3ER)K?S^NQ@$1W$i11DI3v6 z(_?=|F<`O1z(*Ihl^wLTLKh>3Exw%F6#HRWbRox}M6#R7Ogpbny&)-yzRbp)wTuhE z1fl~(bb-AC_Y5QFwimI@Fobk-z)jCein5PE7o%Y=i=y{4;&S2-GZB4OwTaFf2v&kk zHl&V<$8NE5rVJT_=j`;B!?FW`5&r!pu||VKM?Cx?z-F zPR3lgY1)*9>q$AJA`=aab!jHbbWL4{Igs9l_q7&|c3OF0>>TR|1GakBHzxmO%y8`H zH?2>)DoK4ZlDta&oO@jyf_mg_n|y4fJPzFTVl0A7^m2e27jSk8+aG9V&ieJ+S6tS9 z-89uq;tyVz4SrR-ZA|-`lztR(;g40=l`3h|P)B;74Z>vxf*s)3$qrK#j(u(tRyiEo z0&H6wS8r#y5Z7YVOht%;DEGW_>N}puhQqLy)l8u7){)LT0UWBHa<;!$R}te5XrCXT zK*+YYPRvZtPt+LuBukSdicr>7tuHY21#jBQ8?l>=I*Syw*D6elT*U!F8`&dFcOIm& z&+DoJ01Q-N9Mh87f~T}5ZR_W0?R#SElDTMI>A9eQ^_UE+D$ELam7WoH>tga%^v{&- z06aQ>j^8uJ-K@5E;v6?zgyE&1O;ej@CG4Y6KtBgn4YDX3aE~gP>jM;6pL|9Rgb*Jf zv55CjNp}!CFPgm^Gh6$1)4i3{pRt$w)dboi3Tni8Qt&COc{X%z8YH@NC-#Yz7VB1` zkt>UCJ7s9jIZC1Xj~g(b+DNb+xyeuuCJwI$7U2Cpt%OEJIp-tra!yHM`1~boTQ@GWtt+Jku!bs)M2f$72UEsd3;INS8ZH~9;wdB5n zNuR~Et#MvR0{W(tr5BYrhHjgCD-~6?;)Jvo${9=!*l?gbETTJ%Ty*OyxIiZ6Bp&pM z4RS-yHO)dXPGl_FtnYFHCfX-K3A&nd>a7OibecemV1^&q40rbIyHw&HTBv8wb6W+m zMu0cmM)ikL@YI8COPaEbV4m?A4@J@z?kJmn(#b}^YLSh=WoE!W0{9&UWPW<{@dJJF zF>`yTK5X?$m)=_~vwb!8k1qV|^LrYei~}l)LzC>Qa?0zdiVIlOVL3FwFIx|4lW~wi z4SNC@uy5oNyKZ3>88-Rf5dh~=SHbPg)KVu=H$C&-g~dR-u?X|T%QD)gy^m%=d*OSN z%HuWEsXh`18K8urBTzS7iv4}#=+Qola&P!Xkndj^uESvAHm-FSF5Ix?g-y_pa4ngtd@>AsjQMHfgQ+GI1$usCWylyOpyK^Np4r?9ODF8aJ} zf}9K^m~>EA1nm|EWKonlpvD5gPOv_Ecc<>jw;Z-0FA0@|Ox^!~SB?_R!o z_piIxFBCR+>R-Qp`SDwi(lnbPvdOdX^t+z8Rl^TZ8;F2+v6r>5mafbiTDKJ@BB0)g zhg2`gJ1aI4sDf*3(fzYIXqt`WBW^#u`E|L%v-|(6c$0_wH-qCXBD%6WghRJ-@Lth`S$MJho9cR zd-n62*LN@eb@$tUzj^!m+51=j@7;?x|9bY}-QC-l>L=g+ri}O3zx)IK<(2dH$2)c5 zN2wm5_RYLR=WpfFEd=ToeE1jl;TAsKrQr&17c?La3Y?FK&gsB$mKB7Oyvc&PbG0E2 zX0J%Zmz79@k4OlCX{qL~eu&=vrl9-le|~uPUmyPW{nKRqRk_r^zWVp`zy9a*|MVmJ+1kbC*+haX<3KQG_?`17wn-DwfTlabBhXfXgcxv8h)ne#Vl zcto^9^-S|5FC~y^+PUc*gPHO^!xNBRs5PMoqidZ5^#Cxs|2apf;l9KgwJ(5qT(q)o znkstYjSLT)h+&2}f^{83Sj=Ot9LDJ8ctCwK_QKU06}GNr1So`rSqBs`fDy)qw{QOb z=KZ_p@Bioh|9X^iUzO*6zSJMb@Qd$$#tm6W%Sha6_Nd=_RU0apQnxOBA5TRy)2^KH zq$`x@;sw-3XdBVkkH4k*CI7f`Ec-kqbTuN*6*04+ zux-})&mwX*ci04xRmJif^I5u5@F#Wn1Ms-nXMG&>!QZrnV5YZqEgNi8^F52m*~0J5 z7P8dzZR`6w)iDD0L3nInA&%`J=bOd@=^APnY$L970grR+%^TpctTBGu9!uZ!R*#r- z6e2f!tcCswWA<$$qAzbTV!>^wj?W@;fi~-QL@>BwxaUvovs9+;SO?*8v(IuC-t=!g z>eXlp$kOw?5f7U_zi^~49~yx8_m{xBf4L1DHh|x_!_q5R&ce2L3?R4qs|SU0^j)3l zRHY%z*<^I-93mICzHdduUKl&A@5*Drg(K}CJZ^Pa_cH_R8?U`rezX>G7Y(<6NWvb_-EgrqKSb&U8%F zr$rA!F3H$N5>ITP%@(pGK@@`TtAERGIzv7aFrLNZ zEDPh!7NQ;o3*v7)55ao>mO3^)2$8R}kau@KsgLJ=p!C&;_itXl`|$Ule|-Pqmp9M; z@#?3a?%uxm;mzBhU;g{mPap2yJySP-_wv;*@9*CJc=zG$-HYEoeeL1jKKu36+gCr^ z_fk?X@4ulw-^z-FP%d>LwA~(B{wS=V{d{WI>8qLV3ktc|Z=oltlScZr~jggPhl`4Y8 z5VkPAQKQ%a>^Zbg3*V@FR&V?HCA@#B{5mB!P&+&U7=WJvQ;)@I@Bb8FU!zd=M*$mo zQ^E}JJb_rs$WPlU4S4Q>5cVjqr4E?0+G`+$5{rQS0aIDFLBQ~5z?6xd9ua*KFuXlr zsFM&*$ry(9%&SEsbV}o@7fnRcSrCn(o-ykLp)pzbl8yijn*;2Tf!P3@7MpkqFK7a; zIc!iImO(gXoFp0IUYP=S@XChzv2I@uT(estb$Z>{mAVctY|g%b^(p#+mIH`&jXwm| z_HYeePMTTy!x7d)0#byktPv#-nufR}^mOYYD^4o9aAw~)e$6c4JZGoZyx{T&!{1*n z#{#~e39^APxFGrB__D6?hLCIz7cC?nhLC!_6u9Ul7jsr)6muDnF5@}}i49kjpok&{ z>;s&k`YB^@A@Ilh3dbbq3DupUg;sms z>Q_RRbpTujqeBKBJ40|f6?_6NW&kc3j8z^CVDOV_n5Lw5yEyC9xEE1PV^FT(S}Bv` z`&|yQzcYlEWehIIdK4NijKWvI5uH7H(a5FfKbCE_`A| znLq~|;0z%!<_rP)onQs3XUPUvz@@ISJzQ7_3<#cxmkJn!aZ&IR%cLb@pm>aQ>tc8% zAV?^MHolP+&BD%eLQh+lO(3)J=aVOu&H{S?~zB zY;$mZ(Iv6$tKc#a`Sl|g5c{-Ax*;g!*hV<3;{u?7WL=B(A`@MoMi60@cpPF~#?oN; zMH&o|b%is8Ceq+`=-eR=r8|I7#SJrRVUJ3;>uP)`>Se#m^|tZFHAVsm=iJ3 z9WG^*^cAk~+*I7pTp_%&oZBu9$y~L((9OwNRoE>Ds}Jyiv=}jC_N0Q_$IX1_n%N6= z4Fm1QtO+|gk8znsr3}kC?|Pr87@BV&gZd0fziSB+i}0|nh` z3ZjssqMrMjBv~BSz92{h{an{26?!Y5(pAaHv6)Q~Il#8eCB;WEu=bE#L{DPHNG0`(C>#12TT%yok)hL7Na(7GbA zqS(M#XPkDl85 no cost prompt + with patch('builtins.input') as inp: + confirm_cost(None, confirmed=False) + inp.assert_not_called() + + +def test_small_crawl_set_no_prompt(): + with patch('builtins.input') as inp: + confirm_cost(est(5), confirmed=False) + inp.assert_not_called() + + +def test_confirmed_flag_bypasses_prompt(): + with patch('builtins.input') as inp: + confirm_cost(est(None), confirmed=True) + inp.assert_not_called() + + +def test_large_crawl_set_non_tty_aborts(): + with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin: + stdin.isatty.return_value = False + with pytest.raises(SystemExit): + confirm_cost(est(11), confirmed=False) + + +def test_unknown_crawls_non_tty_aborts(): + with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin: + stdin.isatty.return_value = False + with pytest.raises(SystemExit): + confirm_cost(est(None), confirmed=False) + + +def test_tty_yes_proceeds(): + with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch('builtins.input', return_value='y'): + stdin.isatty.return_value = True + confirm_cost(est(None), confirmed=False) # should not raise + + +def test_tty_empty_answer_aborts(): + with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch('builtins.input', return_value=''): + stdin.isatty.return_value = True + with pytest.raises(SystemExit): + confirm_cost(est(None), confirmed=False) + + +def test_tty_no_answer_aborts(): + with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch('builtins.input', return_value='n'): + stdin.isatty.return_value = True + with pytest.raises(SystemExit): + confirm_cost(est(11), confirmed=False) diff --git a/tests/repackage/test_athena_query_builder.py b/tests/repackage/test_athena_query_builder.py new file mode 100644 index 0000000..dd3f2df --- /dev/null +++ b/tests/repackage/test_athena_query_builder.py @@ -0,0 +1,223 @@ +import pytest + +from cdx_toolkit.repackage.sources.sql_base import ( + build_athena_query, + escape_sql_literal, + validate_result_columns, + join_warc_url, +) +from cdx_toolkit.repackage.sources.athena import run_athena_query, iter_range_jobs + + +class _FakeAthenaClient: + """Minimal stand-in for a boto3 Athena client (no AWS, no network).""" + + def __init__(self, raise_on_wait=None): + self._raise_on_wait = raise_on_wait + self.started = [] + self.stopped = [] + + def start_query_execution(self, **kwargs): + self.started.append(kwargs) + return {'QueryExecutionId': 'qid-123'} + + def get_query_execution(self, QueryExecutionId): + if self._raise_on_wait is not None: + raise self._raise_on_wait + return {'QueryExecution': {'Status': {'State': 'SUCCEEDED'}, + 'Statistics': {'DataScannedInBytes': 1000}}} + + def stop_query_execution(self, QueryExecutionId): + self.stopped.append(QueryExecutionId) + + +def test_build_query_hostnames(): + q = build_athena_query(['example.com', 'test.org']) + assert "subset = 'warc'" in q + assert "url_host_tld = 'com'" in q + assert "url_host_tld = 'org'" in q + assert "url_host_name = 'example.com'" in q + assert "url_host_name = 'test.org'" in q + assert 'warc_filename, warc_record_offset, warc_record_length' in q + assert 'crawl IN' not in q + assert 'LIMIT' not in q + + +def test_build_query_domains_only(): + q = build_athena_query(url_host_registered_domains=['example.com']) + assert "url_host_registered_domain = 'example.com'" in q + assert "url_host_tld = 'com'" in q + assert 'url_host_name' not in q + + +def test_build_query_hostnames_and_domains(): + q = build_athena_query(['www.example.com'], url_host_registered_domains=['example.org']) + assert "url_host_name = 'www.example.com'" in q + assert "url_host_registered_domain = 'example.org'" in q + assert "url_host_tld = 'com'" in q + assert "url_host_tld = 'org'" in q + + +def test_build_query_requires_host_or_domain(): + with pytest.raises(ValueError): + build_athena_query() + + +def test_build_query_with_crawls(): + q = build_athena_query(['example.com'], crawls=['CC-MAIN-2025-33', 'CC-MAIN-2025-30']) + assert "crawl IN ('CC-MAIN-2025-33', 'CC-MAIN-2025-30')" in q + + +def test_build_query_limit(): + assert 'LIMIT 10' in build_athena_query(['example.com'], limit=10) + assert 'LIMIT' not in build_athena_query(['example.com'], limit=0) + + +def test_build_query_orders_by_default(): + q = build_athena_query(['example.com']) + assert 'ORDER BY warc_filename, warc_record_offset' in q + + +def test_build_query_order_by_false(): + assert 'ORDER BY' not in build_athena_query(['example.com'], order_by=False) + + +def test_build_query_order_by_precedes_limit(): + q = build_athena_query(['example.com'], limit=10) + assert q.index('ORDER BY') < q.index('LIMIT') + + +def test_build_query_requires_hostnames(): + with pytest.raises(ValueError): + build_athena_query([]) + + +def test_escape_sql_literal_valid(): + assert escape_sql_literal('example.com') == "'example.com'" + assert escape_sql_literal('CC-MAIN-2025-33') == "'CC-MAIN-2025-33'" + + +@pytest.mark.parametrize('bad', ["a';DROP TABLE x;--", "a' OR '1'='1", 'a b', "a'", "', '"]) +def test_escape_sql_literal_rejects_injection(bad): + with pytest.raises(ValueError): + escape_sql_literal(bad) + + +def test_build_query_rejects_injection_hostname(): + with pytest.raises(ValueError): + build_athena_query(["example.com'; DROP TABLE ccindex; --"]) + + +def test_validate_result_columns_ok(): + validate_result_columns(['warc_filename', 'warc_record_offset', 'warc_record_length', 'extra']) + + +def test_validate_result_columns_missing(): + with pytest.raises(ValueError) as e: + validate_result_columns(['warc_filename']) + assert 'warc_record_offset' in str(e.value) + assert 'warc_record_length' in str(e.value) + + +def test_join_warc_url_relative(): + assert ( + join_warc_url('https://data.commoncrawl.org', 'crawl-data/x.warc.gz') + == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' + ) + + +def test_join_warc_url_no_double_slash(): + assert ( + join_warc_url('https://data.commoncrawl.org/', 'crawl-data/x.warc.gz') + == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' + ) + assert join_warc_url('s3://bucket', '/crawl-data/x.warc.gz') == 's3://bucket/crawl-data/x.warc.gz' + + +def test_join_warc_url_empty_prefix(): + assert join_warc_url('', 'crawl-data/x.warc.gz') == 'crawl-data/x.warc.gz' + assert join_warc_url(None, 'crawl-data/x.warc.gz') == 'crawl-data/x.warc.gz' + + +def test_join_warc_url_absolute_filename_passthrough(): + assert join_warc_url('https://data.commoncrawl.org', 's3://cc/x.warc.gz') == 's3://cc/x.warc.gz' + assert join_warc_url('', 'https://host/x.warc.gz') == 'https://host/x.warc.gz' + + +class _FakePaginator: + """Yields Athena get_query_results-style pages (first row is the header).""" + + def __init__(self, pages): + self._pages = pages + + def paginate(self, **kwargs): + return iter(self._pages) + + +class _FakeResultsClient: + def __init__(self, pages): + self._pages = pages + + def get_paginator(self, name): + assert name == 'get_query_results' + return _FakePaginator(self._pages) + + +def _athena_page(columns, *value_rows): + header = {'Data': [{'VarCharValue': c} for c in columns]} + data = [{'Data': [{'VarCharValue': v} for v in row]} for row in value_rows] + return {'ResultSet': {'Rows': [header] + data}} + + +def test_iter_range_jobs_carries_extra_columns(): + pages = [_athena_page( + ['warc_filename', 'warc_record_offset', 'warc_record_length', 'content_languages', 'url'], + ['crawl-data/x.warc.gz', '100', '200', 'eng', 'https://example.com/page'], + )] + client = _FakeResultsClient(pages) + jobs = list(iter_range_jobs(client, 'qid', 'https://data.commoncrawl.org')) + assert len(jobs) == 1 + job = jobs[0] + assert job.url == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' + assert job.offset == 100 and job.length == 200 + assert job.filename == 'crawl-data/x.warc.gz' + # extra carries only the non-required columns (page URL distinct from warc URL) + assert job.extra == {'content_languages': 'eng', 'url': 'https://example.com/page'} + + +def test_iter_range_jobs_no_extra_columns_is_none(): + pages = [_athena_page( + ['warc_filename', 'warc_record_offset', 'warc_record_length'], + ['crawl-data/x.warc.gz', '1', '2'], + )] + jobs = list(iter_range_jobs(_FakeResultsClient(pages), 'qid', 'https://data.commoncrawl.org')) + assert jobs[0].extra is None + + +def test_run_athena_query_logs_sql(caplog): + import logging + client = _FakeAthenaClient() + with caplog.at_level(logging.INFO): + run_athena_query(client, 'SELECT my_special_column FROM ccindex', 'ccindex', 's3://b/out/') + assert 'SELECT my_special_column FROM ccindex' in caplog.text + + +def test_run_athena_query_success_no_cancel(): + client = _FakeAthenaClient() + qid = run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') + assert qid == 'qid-123' + assert client.stopped == [] + + +def test_run_athena_query_cancels_on_interrupt(): + client = _FakeAthenaClient(raise_on_wait=KeyboardInterrupt()) + with pytest.raises(KeyboardInterrupt): + run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') + assert client.stopped == ['qid-123'] + + +def test_run_athena_query_cancels_on_timeout(): + client = _FakeAthenaClient(raise_on_wait=TimeoutError('too slow')) + with pytest.raises(TimeoutError): + run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') + assert client.stopped == ['qid-123'] diff --git a/tests/repackage/test_csv_source.py b/tests/repackage/test_csv_source.py new file mode 100644 index 0000000..b60e112 --- /dev/null +++ b/tests/repackage/test_csv_source.py @@ -0,0 +1,130 @@ +import csv + +import pytest + +from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.sources.csv import RangeJobCsvWriter, CsvSource + + +def test_writer_writes_full_urls_and_reads_them_back(tmp_path): + path = str(tmp_path / 'ranges.csv') + w = RangeJobCsvWriter(path) + w.write(RangeJob(url='s3://commoncrawl/crawl-data/x.warc.gz', offset=5, length=7, filename='crawl-data/x.warc.gz')) + w.close() + + with open(path, newline='') as f: + rows = list(csv.DictReader(f)) + assert rows[0] == { + 'warc_filename': 's3://commoncrawl/crawl-data/x.warc.gz', + 'warc_record_offset': '5', + 'warc_record_length': '7', + } + + # a full URL is used as-is, whatever the prefix + jobs = list(CsvSource(path, 'https://ignored').iter_range_jobs()) + assert jobs[0].url == 's3://commoncrawl/crawl-data/x.warc.gz' + assert jobs[0].offset == 5 and jobs[0].length == 7 + + +def test_csv_source_relative_filename_gets_prefix(tmp_path): + path = str(tmp_path / 'ranges.csv') + with open(path, 'w') as f: + f.write('warc_filename,warc_record_offset,warc_record_length\n') + f.write('crawl-data/x.warc.gz,10,20\n') + f.write('https://example.org/y.warc.gz,1,2\n') + jobs = list(CsvSource(path, 'https://data.commoncrawl.org', sort=False).iter_range_jobs()) + assert [j.url for j in jobs] == [ + 'https://data.commoncrawl.org/crawl-data/x.warc.gz', 'https://example.org/y.warc.gz', + ] + assert jobs[0].filename == 'crawl-data/x.warc.gz' + + +def test_writer_extra_columns_round_trip(tmp_path): + path = str(tmp_path / 'ranges.csv') + w = RangeJobCsvWriter(path) + w.write(RangeJob( + url='https://data.commoncrawl.org/crawl-data/x.warc.gz', + offset=10, length=20, filename='crawl-data/x.warc.gz', + extra={'content_languages': 'eng', 'url': 'https://example.com/page'}, + )) + w.close() + + with open(path, newline='') as f: + reader = csv.DictReader(f) + assert reader.fieldnames == [ + 'warc_filename', 'warc_record_offset', 'warc_record_length', + 'content_languages', 'url', + ] + rows = list(reader) + assert rows[0]['content_languages'] == 'eng' + assert rows[0]['url'] == 'https://example.com/page' + + # extras round-trip back onto RangeJob.extra + jobs = list(CsvSource(path, None).iter_range_jobs()) + assert jobs[0].extra == {'content_languages': 'eng', 'url': 'https://example.com/page'} + assert jobs[0].url == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' + + +def test_writer_empty_writes_base_header(tmp_path): + path = str(tmp_path / 'empty.csv') + w = RangeJobCsvWriter(path) + w.close() + with open(path, newline='') as f: + reader = csv.DictReader(f) + rows = list(reader) + assert reader.fieldnames == ['warc_filename', 'warc_record_offset', 'warc_record_length'] + assert rows == [] + + +def test_csv_source_warc_url_column_is_not_supported(tmp_path): + path = str(tmp_path / 'old.csv') + with open(path, 'w') as f: + f.write('warc_url,warc_record_offset,warc_record_length\n') + f.write('s3://commoncrawl/crawl-data/x.warc.gz,5,7\n') + with pytest.raises(ValueError, match='warc_filename'): + list(CsvSource(path, 'https://x').iter_range_jobs()) + + +def test_csv_source_missing_columns(tmp_path): + path = str(tmp_path / 'bad.csv') + with open(path, 'w') as f: + f.write('foo,bar\n1,2\n') + with pytest.raises(ValueError): + list(CsvSource(path, 'https://x').iter_range_jobs()) + + +def test_csv_source_tsv(tmp_path): + path = str(tmp_path / 'ranges.tsv') + with open(path, 'w') as f: + f.write('warc_filename\twarc_record_offset\twarc_record_length\n') + f.write('crawl-data/x.warc.gz\t10\t20\n') + jobs = list(CsvSource(path, 'https://data.commoncrawl.org').iter_range_jobs()) + assert jobs[0].url == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' + assert jobs[0].offset == 10 and jobs[0].length == 20 + + +def test_csv_source_sorts_by_filename_then_offset(tmp_path): + path = str(tmp_path / 'ranges.csv') + with open(path, 'w') as f: + f.write('warc_filename,warc_record_offset,warc_record_length\n') + # deliberately out of order: file b before a, and offsets descending + f.write('b.warc.gz,50,1\n') + f.write('a.warc.gz,200,1\n') + f.write('a.warc.gz,10,1\n') + jobs = list(CsvSource(path, 'https://data.commoncrawl.org').iter_range_jobs()) + assert [(j.filename, j.offset) for j in jobs] == [ + ('a.warc.gz', 10), ('a.warc.gz', 200), ('b.warc.gz', 50), + ] + + +def test_csv_source_no_sort_preserves_order(tmp_path): + path = str(tmp_path / 'ranges.csv') + with open(path, 'w') as f: + f.write('warc_filename,warc_record_offset,warc_record_length\n') + f.write('b.warc.gz,50,1\n') + f.write('a.warc.gz,200,1\n') + f.write('a.warc.gz,10,1\n') + jobs = list(CsvSource(path, 'https://data.commoncrawl.org', sort=False).iter_range_jobs()) + assert [(j.filename, j.offset) for j in jobs] == [ + ('b.warc.gz', 50), ('a.warc.gz', 200), ('a.warc.gz', 10), + ] diff --git a/tests/repackage/test_data_classes.py b/tests/repackage/test_data_classes.py new file mode 100644 index 0000000..72854d7 --- /dev/null +++ b/tests/repackage/test_data_classes.py @@ -0,0 +1,12 @@ +import pytest +from cdx_toolkit.repackage.data_classes import RangeJob + + +def test_get_s3_bucket_and_key_from_http_job(): + job = RangeJob( + url='http://foo.com/example', + offset=0, + length=10, + ) + with pytest.raises(ValueError): + job.get_s3_bucket_and_key() diff --git a/tests/repackage/test_hf_utils.py b/tests/repackage/test_hf_utils.py new file mode 100644 index 0000000..1759d29 --- /dev/null +++ b/tests/repackage/test_hf_utils.py @@ -0,0 +1,188 @@ +"""Offline unit tests for hf:// bucket URL helpers and the HF reader (no network).""" +import asyncio +import time +from unittest.mock import patch + +import pytest + +from cdx_toolkit.repackage.hf_utils import ( + HfHttpReader, + signed_url_expiry, + is_hf_url, + hf_fs_path, + hf_resolve_url, + retry_after_seconds, +) + + +def test_is_hf_url(): + assert is_hf_url('hf://buckets/commoncrawl/commoncrawl/crawl-data/x.warc.gz') + assert not is_hf_url('s3://commoncrawl/crawl-data/x.warc.gz') + assert not is_hf_url('https://data.commoncrawl.org/x.warc.gz') + assert not is_hf_url('') + assert not is_hf_url(None) + + +def test_hf_fs_path_strips_scheme(): + assert hf_fs_path('hf://buckets/ns/name/a/b.warc.gz') == 'buckets/ns/name/a/b.warc.gz' + + +def test_hf_fs_path_rejects_non_hf(): + with pytest.raises(ValueError): + hf_fs_path('s3://bucket/key') + + +def test_hf_resolve_url_maps_bucket_to_resolve(): + url = 'hf://buckets/commoncrawl/commoncrawl/crawl-data/CC-MAIN-2026-21/seg/x.warc.gz' + assert hf_resolve_url(url) == ( + 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/' + 'crawl-data/CC-MAIN-2026-21/seg/x.warc.gz' + ) + + +def test_hf_resolve_url_rejects_non_bucket(): + # only hf://buckets/// is supported for resolve mapping + with pytest.raises(ValueError): + hf_resolve_url('hf://datasets/foo/bar/x.warc.gz') + with pytest.raises(ValueError): + hf_resolve_url('hf://buckets/onlytwo/segments') + + +def test_retry_after_prefers_retry_after_header(): + assert retry_after_seconds({'Retry-After': '12'}, default=99) == 12.0 + + +def test_retry_after_parses_ratelimit_t_field(): + h = {'RateLimit': '"resolvers";r=0;t=287'} + assert retry_after_seconds(h, default=99) == 287.0 + + +def test_retry_after_falls_back_to_default(): + assert retry_after_seconds({}, default=7.5) == 7.5 + # malformed Retry-After falls through to default (no RateLimit present) + assert retry_after_seconds({'Retry-After': 'soon'}, default=3.0) == 3.0 + + +# --- HfHttpReader: resolve each WARC file once, read the records from the signed CDN URL --- + +WARC_A = 'hf://buckets/commoncrawl/commoncrawl/crawl-data/a.warc.gz' +WARC_B = 'hf://buckets/commoncrawl/commoncrawl/crawl-data/b.warc.gz' +DATA = bytes(range(256)) * 4 + + +class FakeResponse: + def __init__(self, status, headers=None, body=b''): + self.status = status + self.headers = headers or {} + self._body = body + + async def read(self): + return self._body + + async def release(self): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + +class FakeSession: + """Resolve URLs redirect to a signed CDN URL; the CDN serves ranges of DATA.""" + + def __init__(self, ttl=3600, head_statuses=(), get_statuses=()): + self.ttl = ttl + self.head_statuses = list(head_statuses) # statuses returned before the redirect, e.g. 429 + self.get_statuses = list(get_statuses) # statuses returned before serving data, e.g. 403 + self.heads, self.gets = [], [] + + def head(self, url, headers=None, allow_redirects=True): + assert not allow_redirects + self.heads.append((url, headers)) + if self.head_statuses: + return FakeResponse(self.head_statuses.pop(0), {'Retry-After': '0'}) + expires = int(time.time() + self.ttl) + location = f'https://cdn.example/{url.rsplit("/", 1)[-1]}?Expires={expires}&n={len(self.heads)}' + return FakeResponse(302, {'Location': location}) + + def get(self, url, headers=None): + self.gets.append((url, headers)) + if self.get_statuses: + return FakeResponse(self.get_statuses.pop(0)) + start, end = map(int, headers['Range'][len('bytes='):].split('-')) + return FakeResponse(206, body=DATA[start:end + 1]) + + +def _read(reader, url, offset, length): + return reader.ranged_get(url, offset, length, max_attempts=3, base_backoff_seconds=0) + + +def test_signed_url_expiry(): + assert signed_url_expiry('https://cdn.example/x?Expires=1791445181&Signature=s', 600) == 1791445181 + assert abs(signed_url_expiry('https://cdn.example/x', 600) - (time.time() + 600)) < 5 + + +def test_resolves_each_file_once(): + session = FakeSession() + reader = HfHttpReader(session, token='tok') + + async def run(): + return [await _read(reader, url, o, 10) for url, o in [(WARC_A, 0), (WARC_A, 100), (WARC_B, 5), (WARC_A, 50)]] + + assert asyncio.run(run()) == [DATA[0:10], DATA[100:110], DATA[5:15], DATA[50:60]] + assert [url for url, _ in session.heads] == [ + 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/crawl-data/a.warc.gz', + 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/crawl-data/b.warc.gz', + ] + assert session.heads[0][1] == {'Authorization': 'Bearer tok'} + # the token is not sent to the CDN, the URL is signed + assert all(url.startswith('https://cdn.example/') and 'Authorization' not in h for url, h in session.gets) + assert (reader.resolve_count, reader.read_count) == (2, 4) + + +def test_concurrent_reads_share_one_resolve(): + session = FakeSession() + reader = HfHttpReader(session) + + async def run(): + return await asyncio.gather(*[_read(reader, WARC_A, o, 4) for o in range(0, 80, 8)]) + + assert asyncio.run(run()) == [DATA[o:o + 4] for o in range(0, 80, 8)] + assert len(session.heads) == 1 + + +def test_resolves_again_before_expiry(): + session = FakeSession(ttl=HfHttpReader.EXPIRY_MARGIN - 1) # signed URLs that are about to expire + reader = HfHttpReader(session) + + async def run(): + await _read(reader, WARC_A, 0, 4) + await _read(reader, WARC_A, 4, 4) + + asyncio.run(run()) + assert len(session.heads) == 2 + + +def test_resolves_again_when_cdn_rejects_signed_url(): + session = FakeSession(get_statuses=[403]) + reader = HfHttpReader(session) + + assert asyncio.run(_read(reader, WARC_A, 10, 4)) == DATA[10:14] + assert len(session.heads) == 2 # the rejected URL was resolved again + assert session.gets[-1][0].endswith('&n=2') + + +def test_waits_on_resolver_rate_limit(): + session = FakeSession(head_statuses=[429, 429]) + reader = HfHttpReader(session) + + async def no_sleep(_): + return None + + with patch('cdx_toolkit.repackage.hf_utils.asyncio.sleep', side_effect=no_sleep): + assert asyncio.run(_read(reader, WARC_A, 0, 4)) == DATA[0:4] + + assert reader.throttle_count == 2 + assert reader.resolve_count == 1 diff --git a/tests/repackage/test_local_writer.py b/tests/repackage/test_local_writer.py new file mode 100644 index 0000000..c1f3190 --- /dev/null +++ b/tests/repackage/test_local_writer.py @@ -0,0 +1,474 @@ +import asyncio +import pytest +from unittest.mock import patch + +from cdx_toolkit.repackage.local_writer import LocalFileWriter + + +def test_init_default_values(): + """Test initialization with default values.""" + writer = LocalFileWriter('/tmp/test.txt') + assert writer.file_path == '/tmp/test.txt' + assert writer.buffer_size == 8192 + assert writer.mode == 'wb' + assert writer.file_handle is None + assert isinstance(writer.buffer, bytearray) + assert len(writer.buffer) == 0 + + +def test_init_custom_values(): + """Test initialization with custom values.""" + writer = LocalFileWriter('/tmp/test.txt', buffer_size=4096, mode='ab') + assert writer.file_path == '/tmp/test.txt' + assert writer.buffer_size == 4096 + assert writer.mode == 'ab' + assert writer.file_handle is None + assert isinstance(writer.buffer, bytearray) + assert len(writer.buffer) == 0 + + +def test_start_opens_file(tmp_path): + """Test that start() opens the file correctly.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + await writer.start() + + assert writer.file_handle is not None + await writer.close() + + asyncio.run(run_test()) + + +def test_start_with_different_modes(tmp_path): + """Test start() with different file modes.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + + # Test binary write mode + writer = LocalFileWriter(str(temp_file), mode='wb') + await writer.start() + assert writer.file_handle is not None + await writer.close() + + # Test binary append mode + writer = LocalFileWriter(str(temp_file), mode='ab') + await writer.start() + assert writer.file_handle is not None + await writer.close() + + asyncio.run(run_test()) + + +def test_start_creates_directory_if_needed(tmp_path): + """Test that start() works when parent directory exists.""" + + async def run_test(): + subdir = tmp_path / 'subdir' + subdir.mkdir() + temp_file = subdir / 'test.txt' + + writer = LocalFileWriter(str(temp_file)) + await writer.start() + assert writer.file_handle is not None + await writer.close() + + asyncio.run(run_test()) + + +def test_write_small_data_buffers(tmp_path): + """Test writing data that doesn't exceed buffer size.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file), buffer_size=100) + await writer.start() + + test_data = b'Hello, World!' + await writer.write(test_data) + + # Data should be in buffer, not yet written to file + assert len(writer.buffer) == len(test_data) + assert writer.buffer == test_data + + await writer.close() + + # After close, data should be written to file + assert temp_file.read_bytes() == test_data + + asyncio.run(run_test()) + + +def test_write_large_data_triggers_flush(tmp_path): + """Test writing data that exceeds buffer size triggers flush.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + buffer_size = 50 + writer = LocalFileWriter(str(temp_file), buffer_size=buffer_size) + await writer.start() + + # Write data larger than buffer size + test_data = b'x' * (buffer_size + 10) + await writer.write(test_data) + + # Buffer should be empty after automatic flush + assert len(writer.buffer) == 0 + + await writer.close() + + # Data should be written to file + assert temp_file.read_bytes() == test_data + + asyncio.run(run_test()) + + +def test_write_multiple_small_chunks(tmp_path): + """Test writing multiple small chunks that eventually trigger flush.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + buffer_size = 50 + writer = LocalFileWriter(str(temp_file), buffer_size=buffer_size) + await writer.start() + + chunk1 = b'a' * 30 + chunk2 = b'b' * 25 # Total: 55 bytes, exceeds buffer + + await writer.write(chunk1) + assert len(writer.buffer) == 30 + + await writer.write(chunk2) + # Should have triggered flush, buffer should be empty + assert len(writer.buffer) == 0 + + await writer.close() + + assert temp_file.read_bytes() == chunk1 + chunk2 + + asyncio.run(run_test()) + + +def test_write_empty_data(tmp_path): + """Test writing empty data.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + await writer.start() + + await writer.write(b'') + assert len(writer.buffer) == 0 + + await writer.close() + + assert temp_file.read_bytes() == b'' + + asyncio.run(run_test()) + + +def test_write_without_start_graceful_handling(tmp_path): + """Test that writing without calling start() is handled gracefully.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file), buffer_size=10) # Small buffer to force flush + + # This should work fine as long as we don't exceed buffer size + await writer.write(b'small') + assert len(writer.buffer) == 5 + + # When buffer exceeds size, flush is called but does nothing since file_handle is None + # The data stays in buffer instead of being written + await writer.write(b'data that exceeds buffer size') + + # Buffer should contain all the data since flush did nothing + expected_data = b'small' + b'data that exceeds buffer size' + assert writer.buffer == expected_data + + asyncio.run(run_test()) + + +def test_flush_empty_buffer(tmp_path): + """Test flushing when buffer is empty.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + await writer.start() + + # Flush empty buffer should not raise error + await writer._flush() + assert len(writer.buffer) == 0 + + await writer.close() + + asyncio.run(run_test()) + + +def test_flush_without_file_handle(tmp_path): + """Test flushing without file handle.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + writer.buffer.extend(b'test data') + + # Should not raise error, just do nothing + await writer._flush() + assert len(writer.buffer) == len(b'test data') # Buffer unchanged + + asyncio.run(run_test()) + + +def test_close_flushes_remaining_data(tmp_path): + """Test that close() flushes any remaining buffered data.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file), buffer_size=100) + await writer.start() + + test_data = b'This data should be flushed on close' + await writer.write(test_data) + + # Data should still be in buffer + assert len(writer.buffer) == len(test_data) + + await writer.close() + + # Data should now be written to file + assert temp_file.read_bytes() == test_data + + asyncio.run(run_test()) + + +def test_close_without_start(tmp_path): + """Test closing without calling start().""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + + # Should not raise error + await writer.close() + + asyncio.run(run_test()) + + +def test_close_twice(tmp_path): + """Test calling close() multiple times.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + await writer.start() + + await writer.close() + + # Second close should not raise error + await writer.close() + + asyncio.run(run_test()) + + +def test_close_handles_flush_exception(tmp_path): + """Test that close() handles exceptions during flush properly.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + await writer.start() + + # Add some data to buffer + await writer.write(b'test data') + + # Mock flush to raise an exception + with patch.object(writer, '_flush', side_effect=Exception('Flush error')): + with pytest.raises(Exception, match='Flush error'): + await writer.close() + + asyncio.run(run_test()) + + +def test_close_handles_file_close_exception(tmp_path): + """Test that close() handles exceptions during file close.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + await writer.start() + + # Mock file handle close to raise an exception + with patch.object(writer.file_handle, 'close', side_effect=Exception('Close error')): + with pytest.raises(Exception, match='Close error'): + await writer.close() + + asyncio.run(run_test()) + + +def test_large_file_write(tmp_path): + """Test writing a large amount of data.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file), buffer_size=1024) + await writer.start() + + # Write 1MB of data in chunks + chunk_size = 1024 # Make chunk size same as buffer for exact division + total_size = 1024 * 1024 # 1MB + chunk_data = b'x' * chunk_size + + for _ in range(total_size // chunk_size): + await writer.write(chunk_data) + + await writer.close() + + # Verify file size + assert temp_file.stat().st_size == total_size + + asyncio.run(run_test()) + + +def test_binary_data_integrity(tmp_path): + """Test that binary data is written correctly.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + await writer.start() + + # Create binary data with all byte values + binary_data = bytes(range(256)) + await writer.write(binary_data) + + await writer.close() + + assert temp_file.read_bytes() == binary_data + + asyncio.run(run_test()) + + +def test_concurrent_writes(tmp_path): + """Test concurrent write operations.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file), buffer_size=100) + await writer.start() + + # Create multiple write tasks + async def write_chunk(data): + await writer.write(data) + + tasks = [write_chunk(f'chunk{i}'.encode() * 10) for i in range(10)] + + await asyncio.gather(*tasks) + await writer.close() + + # Verify file exists and has data + assert temp_file.exists() + assert temp_file.stat().st_size > 0 + + asyncio.run(run_test()) + + +def test_file_permissions_error(tmp_path): + """Test handling of file permission errors.""" + + async def run_test(): + # Create a file path in a directory we can't write to + readonly_file = tmp_path / 'readonly.txt' + + # Create the file first + readonly_file.write_text('test') + + # Make the file read-only + readonly_file.chmod(0o444) + + writer = LocalFileWriter(str(readonly_file), mode='wb') + + with pytest.raises(PermissionError): + await writer.start() + + asyncio.run(run_test()) + + +def test_nonexistent_directory(): + """Test writing to a file in a nonexistent directory.""" + + async def run_test(): + nonexistent_path = '/nonexistent/directory/file.txt' + writer = LocalFileWriter(nonexistent_path) + + with pytest.raises(FileNotFoundError): + await writer.start() + + asyncio.run(run_test()) + + +def test_context_manager_like_usage(tmp_path): + """Test typical usage pattern similar to context manager.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + writer = LocalFileWriter(str(temp_file)) + + try: + await writer.start() + await writer.write(b'Hello, World!') + await writer.write(b' How are you?') + finally: + await writer.close() + + assert temp_file.read_bytes() == b'Hello, World! How are you?' + + asyncio.run(run_test()) + + +def test_buffer_size_edge_cases(tmp_path): + """Test edge cases with different buffer sizes.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + # Test with buffer size of 1 + writer = LocalFileWriter(str(temp_file), buffer_size=1) + await writer.start() + + await writer.write(b'a') # Should trigger flush immediately + assert len(writer.buffer) == 0 + + await writer.write(b'bc') # Should trigger flush after 'b', leaving 'c' + assert len(writer.buffer) == 0 + + await writer.close() + + assert temp_file.read_bytes() == b'abc' + + asyncio.run(run_test()) + + +def test_append_mode(tmp_path): + """Test append mode functionality.""" + + async def run_test(): + temp_file = tmp_path / 'test.txt' + + # First, write some initial data + temp_file.write_bytes(b'Initial data\n') + + # Now append using LocalFileWriter + writer = LocalFileWriter(str(temp_file), mode='ab') + await writer.start() + + await writer.write(b'Appended data\n') + await writer.close() + + # Verify both pieces of data are present + content = temp_file.read_bytes() + assert content == b'Initial data\nAppended data\n' + + asyncio.run(run_test()) diff --git a/tests/repackage/test_multiprocess_naming.py b/tests/repackage/test_multiprocess_naming.py new file mode 100644 index 0000000..81ab8ae --- /dev/null +++ b/tests/repackage/test_multiprocess_naming.py @@ -0,0 +1,54 @@ +"""Unit tests for multi-process output naming: the file numbers shared by all worker +processes, and copying staged files to their (non-S3) destination. + +No S3 or fetching. +""" +import multiprocessing +import os +from concurrent.futures import ProcessPoolExecutor + +from cdx_toolkit.repackage import multiprocess +from cdx_toolkit.repackage.multiprocess import _copy_staged_files, _output_filename + + +def test_output_filename_local_global_series(): + # No subprefix -> the clean global series matching single-process naming. + assert _output_filename('/out/run', 1) == '/out/run-001.warc.gz' + assert _output_filename('/out/run', 12) == '/out/run-012.warc.gz' + assert _output_filename('/out/run', 2, subprefix='SUB') == '/out/run-SUB-002.warc.gz' + + +def test_output_filename_s3(): + assert _output_filename('s3://b/p/run', 3) == 's3://b/p/run-003.warc.gz' + + +def _take_numbers(n): + return [multiprocess._next_file_sequence(0) for _ in range(n)] + + +def test_file_numbers_are_shared_across_processes(): + ctx = multiprocessing.get_context() + counter = ctx.Value('i', 0) + with ProcessPoolExecutor( + max_workers=3, mp_context=ctx, initializer=multiprocess._init_worker, initargs=(counter,) + ) as ex: + numbers = [n for result in ex.map(_take_numbers, [5, 5, 5]) for n in result] + + # every file number is used exactly once, without gaps + assert sorted(numbers) == list(range(1, 16)) + assert counter.value == 15 + + +def test_copy_staged_files(tmp_path): + staging = tmp_path / 'staging' + staging.mkdir() + for name in ('warc-001.warc.gz', 'warc-002.warc.gz', 'other.txt'): + (staging / name).write_bytes(name.encode()) + dest_dir = tmp_path / 'dest' + dest_dir.mkdir() + + copied = _copy_staged_files(str(staging / 'warc'), str(dest_dir / 'RUN')) + + assert copied == [str(dest_dir / 'RUN-001.warc.gz'), str(dest_dir / 'RUN-002.warc.gz')] + assert sorted(os.listdir(dest_dir)) == ['RUN-001.warc.gz', 'RUN-002.warc.gz'] + assert (dest_dir / 'RUN-002.warc.gz').read_bytes() == b'warc-002.warc.gz' diff --git a/tests/repackage/test_producer.py b/tests/repackage/test_producer.py new file mode 100644 index 0000000..2d6e6dc --- /dev/null +++ b/tests/repackage/test_producer.py @@ -0,0 +1,50 @@ +import asyncio + +from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.sources.base import RangeJobSource +from cdx_toolkit.repackage.warc_filter import WARCFilter, _STOP + + +class FakeSource(RangeJobSource): + def __init__(self, jobs, raise_after=None): + self.jobs = jobs + self.raise_after = raise_after + + def iter_range_jobs(self): + for i, job in enumerate(self.jobs): + if self.raise_after is not None and i == self.raise_after: + raise RuntimeError('boom') + yield job + + +def _jobs(n): + return [ + RangeJob(url=f'https://data.commoncrawl.org/{i}.warc.gz', offset=i, length=1, filename=f'{i}.warc.gz') + for i in range(n) + ] + + +def test_producer_emits_stops_even_when_source_raises(tmp_path): + """Regression: a source that raises mid-iteration must still release the readers.""" + wf = WARCFilter( + source=FakeSource(_jobs(2), raise_after=1), + prefix_path=str(tmp_path), + writer_info={}, + n_parallel_readers=3, + ) + + async def run(): + queue: asyncio.Queue = asyncio.Queue() + try: + await wf._produce_range_jobs(queue) + except RuntimeError: + pass + items = [] + while not queue.empty(): + items.append(await queue.get()) + return items + + items = asyncio.run(run()) + stops = sum(1 for it in items if it is _STOP) + assert wf.num_readers == 3 + assert stops == 3 diff --git a/tests/repackage/test_repackage_cli.py b/tests/repackage/test_repackage_cli.py new file mode 100644 index 0000000..74b797f --- /dev/null +++ b/tests/repackage/test_repackage_cli.py @@ -0,0 +1,205 @@ +"""End-to-end tests of `cdxt repackage` with the async WARC fetcher. + +The captures are the first records of a Common Crawl CDX file (a CDXJ file); the WARC +records are fetched from data.commoncrawl.org (or s3://commoncrawl with AWS credentials). +""" +import gzip +import os +from typing import List, Optional + +import fsspec +import pytest +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main +from cdx_toolkit.record_io import read_records, write_records + +from tests.conftest import requires_aws_s3, TEST_DATA_PATH + + +CDX_PATH = TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz' +METADATA_RECORD_PATH = TEST_DATA_PATH / 'repackage/url_list.txt' + + +def write_captures(tmpdir, n: int = 10, name: str = 'captures.cdxj') -> str: + """The first n captures of the CDX fixture.""" + path = os.path.join(str(tmpdir), name) + with gzip.open(CDX_PATH, 'rt') as f_in, open(path, 'w') as f_out: + for _ in range(n): + f_out.write(f_in.readline()) + return path + + +def read_warc(warc_path): + records = {'warcinfo': [], 'response': [], 'metadata': []} + with fsspec.open(warc_path, 'rb') as stream: + for record in ArchiveIterator(stream): + content = record.content_stream().read().decode('utf-8', errors='ignore') + records.setdefault(record.rec_type, []).append((record, content)) + return records + + +def assert_repackage( + tmpdir, + warc_download_prefix: str, + base_prefix, + extra_args: Optional[List[str]] = None, + warc_filename: str = 'TEST_warc_by_index-001.warc.gz', +): + base_prefix = str(base_prefix) + main( + args=[ + '-v', + 'repackage', + write_captures(tmpdir), + '--no-sort', # keep the order of the CDX file + '--add-metadata-record', + str(METADATA_RECORD_PATH), + f'--prefix={base_prefix}/TEST_warc_by_index', + '--creator=foo', + '--operator=bob', + f'--warc-download-prefix={warc_download_prefix}', + ] + + (extra_args or []) + ) + + # with --processes, each worker writes its own WARC files: read the whole series + fs, fs_prefix = fsspec.url_to_fs(f'{base_prefix}/TEST_warc_by_index') + warc_paths = [fs.unstrip_protocol(p) for p in sorted(fs.glob(fs_prefix + '-*.warc.gz'))] + assert warc_paths[0].endswith('/' + warc_filename) + records = {'warcinfo': [], 'response': [], 'metadata': []} + for warc_path in warc_paths: + file_records = read_warc(warc_path) + # each file is a complete WARC whose warcinfo record names the file itself + (info_record, _), = file_records['warcinfo'] + assert info_record.rec_headers.get('WARC-Filename') == warc_path.rsplit('/', 1)[-1] + for rec_type, recs in file_records.items(): + records[rec_type] += recs + + assert len(records['response']) == 10, 'Invalid record count' + + assert len(records['warcinfo']) == len(warc_paths) + info_record, info_content = records['warcinfo'][0] + assert info_record.rec_headers.get('WARC-Filename') == warc_filename + assert 'operator: bob' in info_content, 'Invalid info record' + + response_contents = [content for _, content in records['response']] + if any(arg.startswith('--processes') for arg in extra_args or []): + # the captures are sharded across processes + assert any('Catalogue en ligne Mission de France' in c for c in response_contents) + else: + assert 'Catalogue en ligne Mission de France' in response_contents[0], 'Invalid response content' + assert 'dojo/dijit/themes/tundra/tundra' in response_contents[9], 'Invalid response content' + + # one metadata record per WARC file + assert len(records['metadata']) == len(warc_paths) + metadata_record, metadata_content = records['metadata'][0] + assert metadata_content[:10] == 'example.co', 'Invalid metadata record' + assert metadata_record.rec_headers.get('WARC-Payload-Digest') == 'sha1:VXA2A5YUS3TAY36AUO6MACRMNOH5RXG2' + with open(METADATA_RECORD_PATH, 'rb') as f: + assert metadata_record.length == len(f.read()) + + +def test_repackage_over_http(tmpdir): + assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=tmpdir) + + +def test_repackage_over_http_with_parallel_readers(tmpdir): + assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=tmpdir, extra_args=['--parallel-readers=3']) + + +def test_repackage_over_http_with_processes(tmpdir): + assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=tmpdir, extra_args=['--processes=2']) + + +@pytest.mark.parametrize('fmt', ['csv', 'tsv', 'jsonl']) +def test_repackage_index_formats(tmpdir, fmt): + """Captures written by `cdxt index` in any format (here: with the columnar index column names for csv).""" + captures = list(read_records(write_captures(tmpdir))) + fields = ['url', 'filename', 'offset', 'length'] + if fmt == 'csv': + for c in captures: + c['warc_filename'], c['warc_record_offset'], c['warc_record_length'] = ( + c.pop('filename'), c.pop('offset'), c.pop('length') + ) + fields = ['url', 'warc_filename', 'warc_record_offset', 'warc_record_length'] + path = os.path.join(str(tmpdir), f'captures.{fmt}') + write_records(captures, path, fmt, fields) + + main(args=['repackage', path, f'--prefix={tmpdir}/TEST']) + + responses = read_warc(os.path.join(str(tmpdir), 'TEST-001.warc.gz'))['response'] + assert len(responses) == 10 + + +def test_repackage_subprefix_and_metadata(tmpdir): + main( + args=[ + 'repackage', + write_captures(tmpdir, n=1), + f'--prefix={tmpdir}/TEST', + '--subprefix=SUB', + '--creator=test_creator', + '--operator=test_operator', + '--is-part-of=PART', + '--description=DESC', + ] + ) + + (_, info), = read_warc(os.path.join(str(tmpdir), 'TEST-SUB-001.warc.gz'))['warcinfo'] + assert 'creator: test_creator' in info + assert 'operator: test_operator' in info + assert 'isPartOf: PART' in info + assert 'description: DESC' in info + + +def test_repackage_without_creator_operator(tmpdir): + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={tmpdir}/TEST_NO_META']) + + (_, info), = read_warc(os.path.join(str(tmpdir), 'TEST_NO_META-001.warc.gz'))['warcinfo'] + assert 'creator:' not in info + assert 'operator:' not in info + + +def test_repackage_glob_input(tmpdir): + write_captures(tmpdir, n=1, name='a.cdxj') + write_captures(tmpdir, n=2, name='b.cdxj') + + main(args=['repackage', f'{tmpdir}/*.cdxj', f'--prefix={tmpdir}/TEST']) + + assert len(read_warc(os.path.join(str(tmpdir), 'TEST-001.warc.gz'))['response']) == 3 + + +def test_repackage_no_input_files_found(tmpdir, capsys): + with pytest.raises(SystemExit) as exc_info: + main(args=['repackage', f'{tmpdir}/nonexistent-*.gz', f'--prefix={tmpdir}/TEST']) + + assert exc_info.value.code == 2 + assert 'no files found' in capsys.readouterr().err + + +@requires_aws_s3 +def test_repackage_over_s3(tmpdir): + assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir) + + +@requires_aws_s3 +def test_repackage_over_s3_to_s3(tmpdir, s3_tmpdir): + assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=s3_tmpdir) + + +@requires_aws_s3 +def test_repackage_over_s3_to_s3_with_parallel_readers(tmpdir, s3_tmpdir): + assert_repackage( + tmpdir, 's3://commoncrawl', base_prefix=s3_tmpdir, extra_args=['--parallel-readers=3', '--is-part-of=foobar'] + ) + + +@requires_aws_s3 +def test_repackage_over_http_to_s3_with_parallel_readers(tmpdir, s3_tmpdir): + assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=s3_tmpdir, extra_args=['--parallel-readers=3']) + + +@requires_aws_s3 +def test_repackage_over_s3_to_local_with_processes(tmpdir): + assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir, extra_args=['--processes=2']) diff --git a/tests/repackage/test_s3_utils.py b/tests/repackage/test_s3_utils.py new file mode 100644 index 0000000..5e159eb --- /dev/null +++ b/tests/repackage/test_s3_utils.py @@ -0,0 +1,180 @@ +import pytest + +import asyncio + + +from cdx_toolkit.repackage.s3_utils import ( + _backoff, + parse_s3_uri, + with_retries, +) + +# the retry tests raise botocore's exceptions (S3 support is optional: cdx_toolkit[s3]) +pytest.importorskip('botocore') +from botocore.exceptions import EndpointConnectionError # noqa: E402 +from botocore.exceptions import ClientError # noqa: E402 + + +def test_backoff(): + """Test _backoff function with exponential backoff and jitter.""" + base_backoff = 1.0 + tol_float = 0.001 # tolerance for float errors + + # Test attempt 1: should be between 0.8 and 1.2 seconds (with jitter) + result1 = _backoff(1, base_backoff) + assert 0.8 <= result1 <= 1.2 + tol_float + + # Test attempt 2: should be between 1.6 and 2.41 seconds (2^1 * base * jitter) + result2 = _backoff(2, base_backoff) + assert 1.6 <= result2 <= 2.4 + tol_float + + # Test attempt 3: should be between 3.2 and 4.8 seconds (2^2 * base * jitter) + result3 = _backoff(3, base_backoff) + assert 3.2 <= result3 <= 4.8 + tol_float + + # Test with different base backoff + base_backoff_small = 0.1 + result_small = _backoff(1, base_backoff_small) + assert 0.08 <= result_small <= 0.12 + tol_float + + # Test minimum backoff (should never be less than 0.05) + very_small_base = 0.001 + result_min = _backoff(1, very_small_base) + assert result_min >= 0.05 + + # Test that backoff increases with attempts + results = [_backoff(i, 0.5) for i in range(1, 6)] + # Generally should increase, though jitter might cause small variations + # Check that the trend is generally increasing + assert results[1] > results[0] * 0.8 # Allow for jitter variation + assert results[2] > results[1] * 0.8 + assert results[3] > results[2] * 0.8 + + +def test_parse_s3_uri(): + """Test parse_s3_uri function for valid and invalid S3 URIs.""" + + # Test valid S3 URIs + bucket, prefix = parse_s3_uri('s3://my-bucket/path/to/file.txt') + assert bucket == 'my-bucket' + assert prefix == 'path/to/file.txt' + + bucket, prefix = parse_s3_uri('s3://test-bucket/folder/subfolder/data.json') + assert bucket == 'test-bucket' + assert prefix == 'folder/subfolder/data.json' + + bucket, prefix = parse_s3_uri('s3://simple/file') + assert bucket == 'simple' + assert prefix == 'file' + + # Test with deep nested paths + bucket, prefix = parse_s3_uri('s3://bucket/a/b/c/d/e/f/file.ext') + assert bucket == 'bucket' + assert prefix == 'a/b/c/d/e/f/file.ext' + + # Test invalid URIs - should raise ValueError + with pytest.raises(ValueError, match='Not an S3 URI'): + parse_s3_uri('http://example.com/path') + + with pytest.raises(ValueError, match='Not an S3 URI'): + parse_s3_uri('ftp://bucket/file') + + with pytest.raises(ValueError, match='Not an S3 URI'): + parse_s3_uri('bucket/file') + + # Test malformed S3 URIs + with pytest.raises(ValueError, match='Malformed S3 URI'): + parse_s3_uri('s3://') + + with pytest.raises(ValueError, match='Malformed S3 URI'): + parse_s3_uri('s3://bucket') + + with pytest.raises(ValueError, match='Malformed S3 URI'): + parse_s3_uri('s3://bucket/') + + with pytest.raises(ValueError, match='Malformed S3 URI'): + parse_s3_uri('s3:///file') + + +def test_with_retries_success(): + """Test with_retries function with successful operation on first attempt.""" + + async def run_test(): + call_count = 0 + + async def successful_coro(): + nonlocal call_count + call_count += 1 + return 'success' + + result = await with_retries(successful_coro, op_name='test_op', max_attempts=3, base_backoff_seconds=0.1) + + assert result == 'success' + assert call_count == 1 + + asyncio.run(run_test()) + + +def test_with_retries_eventual_success(): + """Test with_retries function that succeeds after initial failures.""" + + async def run_test(): + call_count = 0 + + async def eventually_successful_coro(): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise ClientError({'Error': {'Code': 'Throttling'}}, 'test_op') + return 'success' + + result = await with_retries( + eventually_successful_coro, + op_name='test_op', + max_attempts=3, + base_backoff_seconds=0.01, # Very short for testing + ) + + assert result == 'success' + assert call_count == 3 + + asyncio.run(run_test()) + + +def test_with_retries_max_attempts_exceeded(): + """Test with_retries function when max attempts are exceeded.""" + + async def run_test(): + call_count = 0 + + async def failing_coro(): + nonlocal call_count + call_count += 1 + raise EndpointConnectionError(endpoint_url='test') + + with pytest.raises(EndpointConnectionError): + await with_retries(failing_coro, op_name='test_op', max_attempts=2, base_backoff_seconds=0.01) + + assert call_count == 2 + + asyncio.run(run_test()) + + +def test_with_retries_non_retryable_exception(): + """Test with_retries function with non-retryable exceptions.""" + + async def run_test(): + call_count = 0 + + async def failing_coro(): + nonlocal call_count + call_count += 1 + raise ValueError('Non-retryable error') + + with pytest.raises(ValueError): + await with_retries(failing_coro, op_name='test_op', max_attempts=3, base_backoff_seconds=0.01) + + # Should fail immediately without retries + assert call_count == 1 + + asyncio.run(run_test()) diff --git a/tests/repackage/test_s3_writer.py b/tests/repackage/test_s3_writer.py new file mode 100644 index 0000000..38bfd3d --- /dev/null +++ b/tests/repackage/test_s3_writer.py @@ -0,0 +1,311 @@ +import pytest + +import asyncio +from unittest.mock import AsyncMock +from unittest.mock import patch + +from cdx_toolkit.repackage.s3_writer import mpu_abort + + +from cdx_toolkit.repackage.s3_writer import S3ShardWriter + + +def test_shard_writer_start(): + """Test ShardWriter start method.""" + + async def run_test(): + with patch('cdx_toolkit.repackage.s3_writer.mpu_create') as mock_mpu_create: + mock_mpu_create.return_value = 'test-upload-id' + + mock_s3 = AsyncMock() + + writer = S3ShardWriter( + s3_client=mock_s3, + shard_key='test.warc.gz', + dest_bucket='test-bucket', + content_type='application/gzip', + min_part_size=1024, + max_attempts=3, + base_backoff_seconds=0.1, + ) + + await writer.start() + + assert writer.upload_id == 'test-upload-id' + mock_mpu_create.assert_called_once_with( + mock_s3, + 'test-bucket', + 'test.warc.gz', + max_attempts=3, + base_backoff_seconds=0.1, + ) + + asyncio.run(run_test()) + + +def test_shard_writer_write_small_data(): + """Test ShardWriter write method with small data that stays in buffer.""" + + async def run_test(): + mock_s3 = AsyncMock() + + writer = S3ShardWriter( + s3_client=mock_s3, + shard_key='test.warc.gz', + dest_bucket='test-bucket', + content_type='application/gzip', + min_part_size=1024, # 1 KiB + max_attempts=3, + base_backoff_seconds=0.1, + ) + + small_data = b'small test data' + + await writer.write(small_data) + + # Data should be in buffer, no parts uploaded yet + assert len(writer.buffer) == len(small_data) + assert bytes(writer.buffer) == small_data + assert writer.part_number == 1 + assert len(writer.parts) == 0 + + asyncio.run(run_test()) + + +def test_shard_writer_write_large_data(): + """Test ShardWriter write method with large data that triggers part uploads.""" + + async def run_test(): + with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part: + mock_upload_part.return_value = 'test-etag-1' + + mock_s3 = AsyncMock() + + writer = S3ShardWriter( + s3_client=mock_s3, + shard_key='test.warc.gz', + dest_bucket='test-bucket', + content_type='application/gzip', + min_part_size=100, # 100 bytes + max_attempts=3, + base_backoff_seconds=0.1, + ) + writer.upload_id = 'test-upload-id' + + large_data = b'x' * 250 # 250 bytes, should create 2 parts + + await writer.write(large_data) + + # Should have uploaded 2 parts (100 bytes each) with 50 bytes remaining in buffer + assert mock_upload_part.call_count == 2 + assert len(writer.parts) == 2 + assert writer.part_number == 3 # Next part would be #3 + assert len(writer.buffer) == 50 # Remaining bytes + assert bytes(writer.buffer) == b'x' * 50 + + # Verify parts structure + assert writer.parts[0] == {'PartNumber': 1, 'ETag': 'test-etag-1'} + assert writer.parts[1] == {'PartNumber': 2, 'ETag': 'test-etag-1'} + + asyncio.run(run_test()) + + +def test_shard_writer_flush_full_parts(): + """Test ShardWriter _flush_full_parts private method directly.""" + + async def run_test(): + with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part: + mock_upload_part.return_value = 'test-etag-flush' + + mock_s3 = AsyncMock() + writer = S3ShardWriter( + s3_client=mock_s3, + shard_key='test.warc.gz', + dest_bucket='test-bucket', + content_type='application/gzip', + min_part_size=50, # 50 bytes + max_attempts=3, + base_backoff_seconds=0.1, + ) + writer.upload_id = 'test-upload-id' + + # Pre-fill buffer with 150 bytes (should create 3 parts of 50 bytes each) + writer.buffer.extend(b'a' * 150) + + await writer._flush_full_parts() + + # Should have uploaded 3 full parts, no remainder + assert mock_upload_part.call_count == 3 + assert len(writer.parts) == 3 + assert writer.part_number == 4 # Next part would be #4 + assert len(writer.buffer) == 0 # All data flushed + + # Verify all parts were created correctly + for i in range(3): + assert writer.parts[i] == {'PartNumber': i + 1, 'ETag': 'test-etag-flush'} + + asyncio.run(run_test()) + + +def test_shard_writer_close_with_buffer(): + """Test ShardWriter close method with data remaining in buffer.""" + + async def run_test(): + with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part, patch( + 'cdx_toolkit.repackage.s3_writer.mpu_complete' + ) as mock_complete: + mock_upload_part.return_value = 'final-etag' + + mock_s3 = AsyncMock() + + writer = S3ShardWriter( + s3_client=mock_s3, + shard_key='test.warc.gz', + dest_bucket='test-bucket', + content_type='application/gzip', + min_part_size=1000, # Large min size to keep data in buffer + max_attempts=3, + base_backoff_seconds=0.1, + ) + writer.upload_id = 'test-upload-id' + + # Add some data to buffer + remaining_data = b'final chunk data' + writer.buffer.extend(remaining_data) + + await writer.close() + + # Should upload the final part and complete MPU + mock_upload_part.assert_called_once_with( + mock_s3, + 'test-bucket', + 'test.warc.gz', + 'test-upload-id', + 1, # part number + remaining_data, + 3, # max attempts + 0.1, # base backoff + ) + + mock_complete.assert_called_once_with( + mock_s3, + 'test-bucket', + 'test.warc.gz', + 'test-upload-id', + [{'PartNumber': 1, 'ETag': 'final-etag'}], + 3, # max attempts + 0.1, # base backoff + ) + + # Buffer should be cleared + assert len(writer.buffer) == 0 + assert len(writer.parts) == 1 + + asyncio.run(run_test()) + + +def test_shard_writer_close_empty(): + """Test ShardWriter close method with no data (empty buffer, no parts).""" + + async def run_test(): + with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part, patch( + 'cdx_toolkit.repackage.s3_writer.mpu_complete' + ) as mock_complete: + mock_s3 = AsyncMock() + + writer = S3ShardWriter( + s3_client=mock_s3, + shard_key='test.warc.gz', + dest_bucket='test-bucket', + content_type='application/gzip', + min_part_size=1000, + max_attempts=3, + base_backoff_seconds=0.1, + ) + writer.upload_id = 'test-upload-id' + + # No data in buffer, no parts uploaded + await writer.close() + + # Should not upload any parts or complete MPU since there's no data + mock_upload_part.assert_not_called() + mock_complete.assert_not_called() + + # State should remain unchanged + assert len(writer.buffer) == 0 + assert len(writer.parts) == 0 + + asyncio.run(run_test()) + + +def test_shard_writer_close_with_exception(): + """Test ShardWriter close method with exception and abort handling.""" + + async def run_test(): + with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part, patch( + 'cdx_toolkit.repackage.s3_writer.mpu_complete' + ) as mock_complete, patch('cdx_toolkit.repackage.s3_writer.mpu_abort') as mock_abort: + mock_upload_part.return_value = 'error-etag' + mock_complete.side_effect = Exception('Complete failed') + + mock_s3 = AsyncMock() + + writer = S3ShardWriter( + s3_client=mock_s3, + shard_key='test.warc.gz', + dest_bucket='test-bucket', + content_type='application/gzip', + min_part_size=1000, + max_attempts=3, + base_backoff_seconds=0.1, + ) + writer.upload_id = 'test-upload-id' + + # Add some data to buffer to trigger upload and complete + writer.buffer.extend(b'some data') + + # Should raise the exception after attempting abort + with pytest.raises(Exception, match='Complete failed'): + await writer.close() + + # Should have attempted to upload part and complete, then abort on failure + mock_upload_part.assert_called_once() + mock_complete.assert_called_once() + mock_abort.assert_called_once_with(mock_s3, 'test-bucket', 'test.warc.gz', 'test-upload-id') + + asyncio.run(run_test()) + + +def test_mpu_abort_success(): + """Test mpu_abort function with successful abort.""" + + async def run_test(): + mock_s3 = AsyncMock() + bucket = 'test-bucket' + key = 'test-key' + upload_id = 'test-upload-id' + + await mpu_abort(mock_s3, bucket, key, upload_id) + + mock_s3.abort_multipart_upload.assert_called_once_with(Bucket=bucket, Key=key, UploadId=upload_id) + + asyncio.run(run_test()) + + +def test_mpu_abort_with_exception(): + """Test mpu_abort function when abort fails (should catch exception).""" + + async def run_test(): + mock_s3 = AsyncMock() + mock_s3.abort_multipart_upload.side_effect = Exception('S3 error') + + bucket = 'test-bucket' + key = 'test-key' + upload_id = 'test-upload-id' + + # Should not raise exception, should log it instead + await mpu_abort(mock_s3, bucket, key, upload_id) + + mock_s3.abort_multipart_upload.assert_called_once_with(Bucket=bucket, Key=key, UploadId=upload_id) + + asyncio.run(run_test()) diff --git a/tests/repackage/test_sql_index_gated.py b/tests/repackage/test_sql_index_gated.py new file mode 100644 index 0000000..faa11af --- /dev/null +++ b/tests/repackage/test_sql_index_gated.py @@ -0,0 +1,81 @@ +"""Gated end-to-end tests for the SQL indexes (Athena, DuckDB): `cdxt index ... | cdxt repackage`. + +These query only a single crawl partition for a single host (cheap, partition-pruned +-- never an all-crawls scan), write the captures with `cdxt index`, then fetch the +WARC records over HTTP with `cdxt repackage`. They require AWS credentials (and duckdb) +and are skipped in CI. +""" +import json +import os + +import fsspec +import pytest +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main +from tests.conftest import requires_aws_athena, requires_aws_s3, requires_duckdb, TEST_ATHENA_S3_LOCATION + + +CRAWL = 'CC-MAIN-2026-17' +HOST = 'commoncrawl.org' +ATHENA = f'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation={TEST_ATHENA_S3_LOCATION}' + + +@pytest.fixture(autouse=True) +def live_crawl_list_cache(tmp_path, monkeypatch): + """--crawl is resolved with the live list of crawls; cache it apart from the (mocked) list of other tests.""" + monkeypatch.setattr('cdx_toolkit.commoncrawl.CACHE_DIR', str(tmp_path / 'cache') + '/') + + +def _index_and_repackage(tmpdir, index_args): + captures_path = os.path.join(str(tmpdir), 'captures.jsonl') + main(args=['index'] + index_args + ['-o', captures_path]) + + with open(captures_path) as f: + captures = [json.loads(line) for line in f] + assert len(captures) > 0, 'expected at least one capture for the host/crawl' + assert all(HOST in c['url'] for c in captures) + + base_prefix = str(tmpdir) + main(args=['repackage', captures_path, f'--prefix={base_prefix}/TEST_sql']) + + response_count = 0 + with fsspec.open(os.path.join(base_prefix, 'TEST_sql-001.warc.gz'), 'rb') as stream: + for record in ArchiveIterator(stream): + if record.rec_type == 'response': + response_count += 1 + target = record.rec_headers.get_header('WARC-Target-URI') or '' + assert HOST in target, f'unexpected target URI: {target}' + + assert response_count == len(captures), 'every capture should yield a response record' + + +@requires_aws_athena +def test_index_athena_e2e(tmpdir): + _index_and_repackage(tmpdir, [ATHENA, '--crawl', CRAWL, '--hostname', HOST]) + + +@requires_aws_s3 +@requires_duckdb +def test_index_duckdb_e2e(tmpdir): + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--hostname', HOST]) + + +@requires_aws_s3 +@requires_duckdb +def test_index_duckdb_domain_e2e(tmpdir): + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--domain', HOST, '--limit', '20']) + + +@requires_aws_s3 +@requires_duckdb +def test_index_duckdb_sql_e2e(tmpdir): + sql = os.path.join(str(tmpdir), 'query.sql') + with open(sql, 'w') as f: + f.write(f""" + SELECT url, warc_filename, warc_record_offset, warc_record_length, content_languages + FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl={CRAWL}/subset=warc/*.parquet') + WHERE url_host_name = '{HOST}' + LIMIT 10 + """) + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--sql', sql, '--yes']) diff --git a/tests/repackage/test_warc_filter.py b/tests/repackage/test_warc_filter.py new file mode 100644 index 0000000..c22a5dc --- /dev/null +++ b/tests/repackage/test_warc_filter.py @@ -0,0 +1,432 @@ +import asyncio + +from unittest.mock import AsyncMock, patch +from cdx_toolkit.repackage.data_classes import ThroughputTracker +from tests.conftest import TEST_DATA_PATH + +from cdx_toolkit.repackage.warc_filter import WARCFilter +from cdx_toolkit.repackage.sources.records import RecordsSource + +fixture_path = TEST_DATA_PATH / 'repackage' + +# A throwaway source for unit tests that only exercise reader/writer/rotate/log methods. +_FAKE_SOURCE = RecordsSource([], 'https://data.commoncrawl.org') + + +def test_filter_keyboard_interrupt_handling(caplog): + """Test that KeyboardInterrupt is properly handled in the filter method.""" + import logging + + # Set log level to capture WARNING messages + caplog.set_level(logging.WARNING, logger='cdx_toolkit.repackage.warc_filter') + + warc_filter = WARCFilter(source=_FAKE_SOURCE, prefix_path='/fake/prefix', writer_info={'writer_id': 1}) + + # Mock filter_async to raise KeyboardInterrupt + with patch.object(warc_filter, 'filter_async', side_effect=KeyboardInterrupt('Simulated user interrupt')): + # Call the filter method + result = warc_filter.filter() + + # Should return -1 when interrupted + assert result == -1 + + # Should log the warning message + assert 'Interrupted by user.' in caplog.text + + +def test_rotate_files_no_rotation_needed(): + """Test rotate_files when no rotation is needed (file size below limit).""" + + async def run_test(): + warc_filter = WARCFilter( + source=_FAKE_SOURCE, + prefix_path='/fake/prefix', + writer_info={'writer_id': 1}, + max_file_size=1000, # 1KB limit + ) + + mock_writer = AsyncMock() + current_file_sequence = 1 + current_file_size = 500 # 500 bytes + added_byte_size = 200 # Adding 200 bytes, total would be 700 (below limit) + + # Call rotate_files + result_writer, result_sequence, result_size = await warc_filter.rotate_files( + writer=mock_writer, + current_file_sequence=current_file_sequence, + current_file_size=current_file_size, + added_byte_size=added_byte_size, + output_path_prefix='/fake/output', + max_attempts=3, + base_backoff_seconds=1.0, + min_part_size=1024, + writer_info={'writer_id': 1}, + ) + + # Should return original values since no rotation occurred + assert result_writer == mock_writer + assert result_sequence == current_file_sequence + assert result_size == current_file_size + + # Writer should not be closed + mock_writer.close.assert_not_called() + + asyncio.run(run_test()) + + +def test_rotate_files_rotation_needed_without_resource_records(): + """Test rotate_files when rotation is needed and no resource records to write.""" + + async def run_test(): + warc_filter = WARCFilter( + source=_FAKE_SOURCE, + prefix_path='/fake/prefix', + writer_info={'writer_id': 1}, + max_file_size=1000, # 1KB limit + write_paths_as_metadata_records=None, # No resource records + ) + + mock_writer = AsyncMock() + mock_new_writer = AsyncMock() + current_file_sequence = 1 + current_file_size = 800 # 800 bytes + added_byte_size = 300 # Adding 300 bytes, total would be 1100 (above limit) + + # Mock create_new_writer_with_header + with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: + mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') # (writer, header_size, warcinfo_id) + + # Call rotate_files + result_writer, result_sequence, result_size = await warc_filter.rotate_files( + writer=mock_writer, + current_file_sequence=current_file_sequence, + current_file_size=current_file_size, + added_byte_size=added_byte_size, + output_path_prefix='/fake/output', + max_attempts=3, + base_backoff_seconds=1.0, + min_part_size=1024, + writer_info={'writer_id': 1}, + ) + + # Should have rotated + assert result_writer == mock_new_writer + assert result_sequence == current_file_sequence + 1 # Incremented + assert result_size == 150 # Header size only + + # Old writer should be closed + mock_writer.close.assert_called_once() + + # New writer should be created + mock_create.assert_called_once_with( + sequence=current_file_sequence + 1, + output_path_prefix='/fake/output', + max_attempts=3, + base_backoff_seconds=1.0, + min_part_size=1024, + writer_info={'writer_id': 1}, + ) + + asyncio.run(run_test()) + + +def test_rotate_files_rotation_needed_with_metadata_records(): + """Test rotate_files when rotation is needed and metadata records need to be written.""" + + async def run_test(): + warc_filter = WARCFilter( + source=_FAKE_SOURCE, + prefix_path='/fake/prefix', + writer_info={'writer_id': 1}, + max_file_size=1000, # 1KB limit + write_paths_as_metadata_records=['/fake/resource1.txt', '/fake/resource2.txt'], + ) + + mock_writer = AsyncMock() + mock_new_writer = AsyncMock() + current_file_sequence = 1 + current_file_size = 800 # 800 bytes + added_byte_size = 300 # Adding 300 bytes, total would be 1100 (above limit) + + # Mock create_new_writer_with_header + with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: + mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') + + # Mock write_metadata_records + with patch.object(warc_filter, 'write_metadata_records', return_value=75) as mock_write_resources: + # Call rotate_files + result_writer, result_sequence, result_size = await warc_filter.rotate_files( + writer=mock_writer, + current_file_sequence=current_file_sequence, + current_file_size=current_file_size, + added_byte_size=added_byte_size, + output_path_prefix='/fake/output', + max_attempts=3, + base_backoff_seconds=1.0, + min_part_size=1024, + writer_info={'writer_id': 1}, + ) + + # Should have rotated + assert result_writer == mock_new_writer + assert result_sequence == current_file_sequence + 1 + assert result_size == 150 + 75 # Header size + resource records size + + # Old writer should be closed + mock_writer.close.assert_called_once() + + # New writer should be created + mock_create.assert_called_once() + + # Resource records should be written + mock_write_resources.assert_called_once_with(mock_new_writer, warcinfo_id='warcinfo-123') + + asyncio.run(run_test()) + + +def test_rotate_files_no_max_file_size_set(): + """Test rotate_files when max_file_size is not set (None).""" + + async def run_test(): + warc_filter = WARCFilter( + source=_FAKE_SOURCE, + prefix_path='/fake/prefix', + writer_info={'writer_id': 1}, + max_file_size=None, # No limit + ) + + mock_writer = AsyncMock() + current_file_sequence = 1 + current_file_size = 999999999 # Very large file + added_byte_size = 999999999 # Very large addition + + # Call rotate_files + result_writer, result_sequence, result_size = await warc_filter.rotate_files( + writer=mock_writer, + current_file_sequence=current_file_sequence, + current_file_size=current_file_size, + added_byte_size=added_byte_size, + output_path_prefix='/fake/output', + max_attempts=3, + base_backoff_seconds=1.0, + min_part_size=1024, + writer_info={'writer_id': 1}, + ) + + # Should not rotate regardless of size + assert result_writer == mock_writer + assert result_sequence == current_file_sequence + assert result_size == current_file_size + + # Writer should not be closed + mock_writer.close.assert_not_called() + + asyncio.run(run_test()) + + +def test_rotate_files_edge_case_exact_limit(): + """Test rotate_files when the total size exactly equals the limit.""" + + async def run_test(): + warc_filter = WARCFilter( + source=_FAKE_SOURCE, + prefix_path='/fake/prefix', + writer_info={'writer_id': 1}, + max_file_size=1000, # 1KB limit + ) + + mock_writer = AsyncMock() + current_file_sequence = 1 + current_file_size = 700 # 700 bytes + added_byte_size = 300 # Adding 300 bytes, total would be exactly 1000 + + # Call rotate_files + result_writer, result_sequence, result_size = await warc_filter.rotate_files( + writer=mock_writer, + current_file_sequence=current_file_sequence, + current_file_size=current_file_size, + added_byte_size=added_byte_size, + output_path_prefix='/fake/output', + max_attempts=3, + base_backoff_seconds=1.0, + min_part_size=1024, + writer_info={'writer_id': 1}, + ) + + # Should not rotate when exactly at limit (only rotate when > limit) + assert result_writer == mock_writer + assert result_sequence == current_file_sequence + assert result_size == current_file_size + + # Writer should not be closed + mock_writer.close.assert_not_called() + + asyncio.run(run_test()) + + +def test_rotate_files_edge_case_just_over_limit(): + """Test rotate_files when the total size is just 1 byte over the limit.""" + + async def run_test(): + warc_filter = WARCFilter( + source=_FAKE_SOURCE, + prefix_path='/fake/prefix', + writer_info={'writer_id': 1}, + max_file_size=1000, # 1KB limit + ) + + mock_writer = AsyncMock() + mock_new_writer = AsyncMock() + current_file_sequence = 1 + current_file_size = 700 # 700 bytes + added_byte_size = 301 # Adding 301 bytes, total would be 1001 (1 byte over) + + # Mock create_new_writer_with_header + with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: + mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') + + # Call rotate_files + result_writer, result_sequence, result_size = await warc_filter.rotate_files( + writer=mock_writer, + current_file_sequence=current_file_sequence, + current_file_size=current_file_size, + added_byte_size=added_byte_size, + output_path_prefix='/fake/output', + max_attempts=3, + base_backoff_seconds=1.0, + min_part_size=1024, + writer_info={'writer_id': 1}, + ) + + # Should rotate when just over limit + assert result_writer == mock_new_writer + assert result_sequence == current_file_sequence + 1 + assert result_size == 150 + + # Old writer should be closed + mock_writer.close.assert_called_once() + + asyncio.run(run_test()) + + +def test_rotate_files_kwargs_passed_through(): + """Test that all kwargs are properly passed to create_new_writer_with_header.""" + + async def run_test(): + warc_filter = WARCFilter( + source=_FAKE_SOURCE, prefix_path='/fake/prefix', writer_info={'writer_id': 1}, max_file_size=1000 + ) + + mock_writer = AsyncMock() + mock_new_writer = AsyncMock() + + # Mock create_new_writer_with_header + with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: + mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') + + # Call rotate_files with various kwargs + await warc_filter.rotate_files( + writer=mock_writer, + current_file_sequence=1, + current_file_size=800, + added_byte_size=300, + output_path_prefix='/custom/output', + max_attempts=5, + base_backoff_seconds=2.5, + min_part_size=2048, + writer_info={'custom': 'info'}, + warc_version='1.1', + gzip=True, + custom_param='custom_value', + ) + + # Verify all kwargs are passed through + mock_create.assert_called_once_with( + sequence=2, # incremented from 1 + output_path_prefix='/custom/output', + max_attempts=5, + base_backoff_seconds=2.5, + min_part_size=2048, + writer_info={'custom': 'info'}, + warc_version='1.1', + gzip=True, + custom_param='custom_value', + ) + + asyncio.run(run_test()) + + +def test_rotate_files_logging(caplog): + """Test that rotation logs the appropriate message.""" + import logging + + async def run_test(): + # Set log level to capture INFO messages + caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.warc_filter') + + warc_filter = WARCFilter( + source=_FAKE_SOURCE, prefix_path='/fake/prefix', writer_info={'writer_id': 1}, max_file_size=1000 + ) + + mock_writer = AsyncMock() + mock_new_writer = AsyncMock() + + # Mock create_new_writer_with_header + with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: + mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') + + # Call rotate_files to trigger rotation + await warc_filter.rotate_files( + writer=mock_writer, + current_file_sequence=5, + current_file_size=800, + added_byte_size=300, + output_path_prefix='/fake/output', + max_attempts=3, + base_backoff_seconds=1.0, + min_part_size=1024, + writer_info={'writer_id': 1}, + ) + + # Check that the rotation log message was written + assert 'Rotated to new WARC file sequence 6 due to size limit' in caplog.text + + asyncio.run(run_test()) + + +def test_log_writer(caplog): + """Test log writer.""" + import logging + caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.warc_filter') + + warc_filter = WARCFilter( + source=_FAKE_SOURCE, + prefix_path='/fake/prefix', + writer_info={'writer_id': 1}, + log_every_n=2, + ) + tracker = ThroughputTracker() + warc_filter.log_writer(0, tracker) + warc_filter.log_writer(1, tracker) + warc_filter.log_writer(2, tracker) + + assert caplog.text.count('WARC Writer') == 2 + + +def test_log_reader(caplog): + """Test log reader.""" + import logging + caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.warc_filter') + + warc_filter = WARCFilter( + source=_FAKE_SOURCE, + prefix_path='/fake/prefix', + writer_info={'writer_id': 1}, + log_every_n=2, + ) + tracker = ThroughputTracker() + warc_filter.log_reader(1, 0, tracker) + warc_filter.log_reader(1, 1, tracker) + warc_filter.log_reader(1, 2, tracker) + + assert caplog.text.count('WARC Reader 1') == 2 diff --git a/tests/repackage/test_warc_from_fs.py b/tests/repackage/test_warc_from_fs.py new file mode 100644 index 0000000..aa23249 --- /dev/null +++ b/tests/repackage/test_warc_from_fs.py @@ -0,0 +1,53 @@ +from tests.conftest import requires_aws_s3 +from cdx_toolkit.warc import fetch_warc_record + + +def test_fetch_warc_record_from_http(): + encoding = 'utf-8' + capture = { + 'url': 'https://bibliotheque.missiondefrance.fr/index.php?lvl=bulletin_display&id=319', + 'mime': 'text/html', + 'mime-detected': 'application/xhtml+xml', + 'status': '200', + 'digest': 'D5K3FUWDRAOMMTJC2CTWV7L2ABFIJ5BP', + 'length': '9754', + 'offset': '111440525', + 'filename': 'crawl-data/CC-MAIN-2024-30/segments/1720763514759.37/warc/CC-MAIN-20240716142214-20240716172214-00337.warc.gz', # noqa: E501 + 'charset': 'UTF-8', + 'languages': 'fra', + 'timestamp': '20240716153155', + } + warc_download_prefix = 'https://data.commoncrawl.org' + + record = fetch_warc_record(capture, warc_download_prefix) + record_content = record.content_stream().read().decode(encoding, errors='ignore') + + assert record.rec_type == 'response' + assert record.length == 75825 + assert 'Catalogue en ligne Mission de France' in record_content + + +@requires_aws_s3 +def test_fetch_warc_record_from_s3(): + encoding = 'utf-8' + capture = { + 'url': 'https://bibliotheque.missiondefrance.fr/index.php?lvl=bulletin_display&id=319', + 'mime': 'text/html', + 'mime-detected': 'application/xhtml+xml', + 'status': '200', + 'digest': 'D5K3FUWDRAOMMTJC2CTWV7L2ABFIJ5BP', + 'length': '9754', + 'offset': '111440525', + 'filename': 'crawl-data/CC-MAIN-2024-30/segments/1720763514759.37/warc/CC-MAIN-20240716142214-20240716172214-00337.warc.gz', # noqa: E501 + 'charset': 'UTF-8', + 'languages': 'fra', + 'timestamp': '20240716153155', + } + warc_download_prefix = 's3://commoncrawl' + + record = fetch_warc_record(capture, warc_download_prefix) + record_content = record.content_stream().read().decode(encoding, errors='ignore') + + assert record.rec_type == 'response' + assert record.length == 75825 + assert 'Catalogue en ligne Mission de France' in record_content diff --git a/tests/repackage/test_warc_utils.py b/tests/repackage/test_warc_utils.py new file mode 100644 index 0000000..d34c6e0 --- /dev/null +++ b/tests/repackage/test_warc_utils.py @@ -0,0 +1,14 @@ +from cdx_toolkit.repackage.warc_utils import get_metadata_record_from_path +from tests.conftest import TEST_DATA_PATH + + +def test_get_metadata_record_from_path(): + file_path = TEST_DATA_PATH / 'repackage/url_list.txt' + record = get_metadata_record_from_path(file_path, warcinfo_id="abc123") + + assert record.content_type == 'text/plain' + + record_headers = dict(record.rec_headers.headers) + + assert record_headers["WARC-Warcinfo-ID"] == "abc123", "Invalid Warcinfo-ID" + assert record_headers["WARC-Payload-Digest"] == "sha1:VXA2A5YUS3TAY36AUO6MACRMNOH5RXG2", "Invalid payload digest" diff --git a/tests/repackage/test_warc_writer.py b/tests/repackage/test_warc_writer.py new file mode 100644 index 0000000..910411c --- /dev/null +++ b/tests/repackage/test_warc_writer.py @@ -0,0 +1,142 @@ +from io import BytesIO +import fsspec +import pytest +import cdx_toolkit + +from tests.conftest import TEST_DATA_PATH, requires_aws_s3 + +from warcio import WARCWriter +from warcio.archiveiterator import ArchiveIterator + + +fixture_path = TEST_DATA_PATH / 'repackage' + + +@pytest.mark.parametrize( + 'prefix,gzip', + [ + pytest.param('test-prefix', False, id='File name prefix on local'), + pytest.param('test-prefix', True, id='File name prefix on local with gzip'), + # raised FileNotFound error (parent dir does not exist) + # pytest.param("test-prefix-folder/file-prefix", None, id="Folder as prefix"), + ], +) +def test_write_to_local(prefix, gzip, tmpdir): + info = { + 'software': 'pypi_cdx_toolkit/test', + 'description': 'test', + 'format': 'WARC file version 1.0', + } + encoding = 'utf-8' + full_prefix = str(tmpdir) + '/' + prefix + fs, fs_prefix_path = fsspec.url_to_fs(full_prefix) + + writer = cdx_toolkit.warc.get_writer(full_prefix, None, info, gzip=gzip) + + # single record + input_resource_record_text = 'foo bar text' + writer.write_record( + WARCWriter(None).create_warc_record( + uri='foo/bar', + record_type='resource', + payload=BytesIO(input_resource_record_text.encode(encoding)), + warc_content_type='text/plain', + ) + ) + writer.close() + + # Check that WARC file was created + warc_path = fs_prefix_path + '-000000.extracted.warc' + if gzip: + warc_path += '.gz' + + assert fs.exists(warc_path) + + # Validate that creator/operator are not in warcinfo record + info_record = None + resource_record = None + with open(warc_path, 'rb') as stream: + for record in ArchiveIterator(stream): + if record.rec_type == 'warcinfo': + info_record = record.content_stream().read().decode(encoding) + + if record.rec_type == 'resource': + resource_record = record.content_stream().read().decode(encoding) + break + + assert resource_record is not None + assert info_record is not None + + assert 'description: test' in info_record + assert resource_record == input_resource_record_text + + +@requires_aws_s3 +def test_write_to_s3(s3_tmpdir): + info = { + 'software': 'pypi_cdx_toolkit/test', + 'description': 'test', + 'format': 'WARC file version 1.0', + } + encoding = 'utf-8' + + fs, fs_prefix_path = fsspec.url_to_fs(s3_tmpdir) + + writer = cdx_toolkit.warc.get_writer(s3_tmpdir, None, info) + + # single record + input_resource_record_text = 'foo bar text' + writer.write_record( + WARCWriter(None).create_warc_record( + uri='foo/bar', + record_type='resource', + payload=BytesIO(input_resource_record_text.encode(encoding)), + warc_content_type='text/plain', + ) + ) + writer.close() + + # Check that WARC file was created + warc_path = fs_prefix_path + '-000000.extracted.warc.gz' + assert fs.exists(warc_path) + + # Validate that creator/operator are not in warcinfo record + info_record = None + resource_record = None + with fs.open(warc_path, 'rb') as stream: + for record in ArchiveIterator(stream): + if record.rec_type == 'warcinfo': + info_record = record.content_stream().read().decode(encoding) + + if record.rec_type == 'resource': + resource_record = record.content_stream().read().decode(encoding) + break + + assert resource_record is not None + assert info_record is not None + + assert 'description: test' in info_record + assert resource_record == input_resource_record_text + + +def test_warc_info(): + warc_version = '1.0' + gzip = False + file_handler = BytesIO() + filename = 'foo.warc' + + info = { + 'software': 'pypi_cdx_toolkit/123', + 'isPartOf': 'bar', + 'description': 'warc extraction based on CDX generated with: xx', + 'format': 'WARC file version 1.0', + } + + writer = WARCWriter(file_handler, gzip=gzip, warc_version=warc_version) + warcinfo = writer.create_warcinfo_record(filename, info) + + writer.write_record(warcinfo) + + file_value = file_handler.getvalue().decode('utf-8') + + assert 'pypi_cdx_toolkit/123' in file_value diff --git a/tests/test_cli.py b/tests/test_cli.py index 5d385f8..e1f1508 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -35,7 +35,7 @@ def slow_ci(): @conditional_mock_responses def test_basics(capsys): - args = '--cc --limit 10 iter commoncrawl.org/*'.split() + args = 'index cc --limit 10 --url commoncrawl.org/*'.split() main(args=args) out, err = capsys.readouterr() @@ -48,7 +48,7 @@ def test_basics(capsys): @conditional_mock_responses def test_basics_2(capsys): - args = '--crawl 2 --limit 10 iter commoncrawl.org/*'.split() + args = 'index cc --crawl 2 --limit 10 --url commoncrawl.org/*'.split() main(args=args) out, err = capsys.readouterr() @@ -60,9 +60,7 @@ def test_basics_2(capsys): def multi_helper(t, capsys, caplog): - inputs = t[0] - outputs = t[1] - cmdline = '{} {} {} {}'.format(inputs['service'], inputs['mods'], inputs['cmd'], inputs['rest']) + cmdline, outputs = t args = cmdline.split() if 'exception' in outputs: @@ -97,442 +95,327 @@ def multi_helper(t, capsys, caplog): @conditional_mock_responses def test_multi_cc1(capsys, caplog): # this is the test case before slow_ci -> break - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrep': 'commoncrawl.org'}, - ] + testdata = ('index cc --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': 'commoncrawl.org'}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_1(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 11', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 11, 'linefgrep': 'commoncrawl.org'}, - ] + testdata = ('index cc --limit 11 --url commoncrawl.org/*', {'count': 11, 'linefgrep': 'commoncrawl.org'}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_2(capsys, caplog): - testdata = [ - { - 'service': '--crawl 1', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/thisurlneverdidexist', - }, - {'count': 0}, - ] # runs slowly if we don't limit crawl to 1 + # runs slowly if we don't limit crawl to 1 + testdata = ('index cc --crawl 1 --limit 10 --url commoncrawl.org/thisurlneverdidexist', {'count': 0}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_3(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--cc-mirror https://index.commoncrawl.org/ --limit 11', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, + testdata = ( + 'index cc --cc-mirror https://index.commoncrawl.org/ --limit 11 --url commoncrawl.org/*', {'count': 11, 'linefgrep': 'commoncrawl.org'}, - ] + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_4(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/* --all-fields', - }, - {'count': 10, 'linefgrep': 'digest '}, - ] + # all fields are output by default + testdata = ('index cc --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': '"digest": '}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_5(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/* --fields=digest,length,offset --csv', - }, + testdata = ( + 'index cc --limit 10 --url commoncrawl.org/* --fields digest,length,offset --output-format csv', {'count': 11, 'csv': True}, - ] + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_6(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10 --filter=status:200', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrep': 'status 200'}, - ] + testdata = ( + 'index cc --limit 10 --filter=status:200 --url commoncrawl.org/*', + {'count': 10, 'linefgrep': '"status": "200"'}, + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_7(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10 --filter=!status:200 --filter=!status:404', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrepv': 'status 200'}, - ] + testdata = ( + 'index cc --limit 10 --filter=!status:200 --filter=!status:404 --url commoncrawl.org/*', + {'count': 10, 'linefgrepv': '"status": "200"'}, + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_8(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10 --to=2017', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrep': 'timestamp 2017'}, - ] + testdata = ( + 'index cc --limit 10 --to=2017 --url commoncrawl.org/*', + {'count': 10, 'linefgrep': '"timestamp": "2017'}, + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_9(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10 --from=2017 --to=2017', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrep': 'timestamp 2017'}, - ] + testdata = ( + 'index cc --limit 10 --from=2017 --to=2017 --url commoncrawl.org/*', + {'count': 10, 'linefgrep': '"timestamp": "2017'}, + ) multi_helper(testdata, capsys, caplog) @conditional_mock_responses def test_multi_cc2(capsys, caplog): # this is the test case before slow_ci -> break - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 3 --get --closest=20170615', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 3, 'linefgrep': 'timestamp 20170'}, - ] # data-dependent, and kinda broken + testdata = ( + 'index cc --limit 3 --closest=20170615 --url commoncrawl.org/*', + {'count': 3, 'linefgrep': '"timestamp": "20170'}, + ) # data-dependent, and kinda broken multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_1(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 3 --get --filter status:200 --closest=20170615', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 3, 'linefgrep': 'timestamp 20170'}, - ] # data-dependent, and kinda broken + testdata = ( + 'index cc --limit 3 --filter status:200 --closest=20170615 --url commoncrawl.org/*', + {'count': 3, 'linefgrep': '"timestamp": "20170'}, + ) # data-dependent, and kinda broken multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_2(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--get --closest=20170615', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/never-existed', - }, - {'count': 0}, - ] + testdata = ('index cc --closest=20170615 --url commoncrawl.org/never-existed', {'count': 0}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_3(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/* --csv', - }, - {'count': 11, 'csv': True}, - ] + testdata = ('index cc --limit 10 --url commoncrawl.org/* --output-format csv', {'count': 11, 'csv': True}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_4(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/* --jsonl', - }, - {'count': 10, 'jsonl': True}, - ] + testdata = ('index cc --limit 10 --url commoncrawl.org/* --output-format jsonl', {'count': 10, 'jsonl': True}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_5(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '-v -v --limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'debug': 5}, - ] + testdata = ('index cc -v -v --limit 10 --url commoncrawl.org/*', {'count': 10, 'debug': 5}) multi_helper(testdata, capsys, caplog) @conditional_mock_responses def test_multi_ia(capsys, caplog): - testdata = [ - {'service': '--ia', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - {'count': 10, 'linefgrep': 'commoncrawl.org'}, - ] + testdata = ('index ia --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': 'commoncrawl.org'}) # Disabled: minimize IA for ratelimit purposes - # [{'service': '--ia', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/thisurlneverdidexist'}, - # {'count': 0}], - # [{'service': '--ia', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/* --all-fields'}, - # {'count': 10, 'linefgrep': 'mime ', 'linefgrepv': 'original '}], # both of these are renamed fields - # [{'service': '--ia', 'mods': '--get --limit 4 --closest=20170615', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - # {'count': 4, 'linefgrep': 'timestamp '}], # returns 2008 ?! bug probably on my end - # [{'service': '--ia', 'mods': '-v -v --limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - # {'count': 10, 'debug': 5}] + # ('index ia --limit 10 --url commoncrawl.org/thisurlneverdidexist', {'count': 0}), + # ('index ia --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': '"mime": '}), # renamed field + # ('index ia --limit 4 --closest=20170615 --url commoncrawl.org/*', + # {'count': 4, 'linefgrep': '"timestamp": '}), # returns 2008 ?! bug probably on my end + # ('index ia -v -v --limit 10 --url commoncrawl.org/*', {'count': 10, 'debug': 5}), multi_helper(testdata, capsys, caplog) def test_multi_misc_not_ia(capsys, caplog): # this is the test case before slow_ci -> break - testdata = [ - { - 'service': '-v -v --source https://web.arc4567hive.org/cdx/search/cdx', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, + testdata = ( + '-v -v index https://web.arc4567hive.org/cdx/search/cdx --limit 10 --url commoncrawl.org/*', {'exception': ValueError}, - ] + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_misc_not_ia_slow_1(capsys, caplog): - testdata = [ - { - 'service': '-v -v --source https://example.com/404', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, + testdata = ( + '-v -v index https://example.com/404 --limit 10 --url commoncrawl.org/*', {'exception': ValueError}, - ] + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_misc_not_ia_slow_2(capsys, caplog): - testdata = [ - {'service': '--crawl 1,1', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - {'exception': ValueError}, - ] + testdata = ('index cc --crawl 1,1 --limit 10 --url commoncrawl.org/*', {'exception': ValueError}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_misc_not_ia_slow_3(capsys, caplog): - testdata = [ - {'service': '--crawl 1,CC-MAIN-2024', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - {'exception': ValueError}, - ] + testdata = ('index cc --crawl 1,CC-MAIN-2024 --limit 10 --url commoncrawl.org/*', {'exception': ValueError}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_misc_not_ia_slow_4(capsys, caplog): - testdata = [ - {'service': '--cc', 'mods': '--limit 10', 'cmd': 'size', 'rest': 'commoncrawl.org/*'}, - {'count': 1, 'is_int': True}, - ] - multi_helper(testdata, capsys, caplog) - - -@pytest.mark.skipif(slow_ci(), reason='Slow CI') -@conditional_mock_responses -def test_multi_misc_not_ia_slow_5(capsys, caplog): - testdata = [ - {'service': '--cc', 'mods': '--limit 10', 'cmd': 'size', 'rest': '--details commoncrawl.org/*'}, - {'count': 2}, - ] + testdata = ('index cc --limit 10 --url commoncrawl.org/* --count', {'count': 1, 'is_int': True}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') -@conditional_mock_responses def test_multi_misc_not_ia_slow_6(capsys, caplog): - testdata = [ - {'service': '', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - {'exception': ValueError}, - ] - multi_helper(testdata, capsys, caplog) + # INDEX is required + with pytest.raises(SystemExit): + main(args='index --limit 10 --url commoncrawl.org/*'.split()) @conditional_mock_responses def test_multi_misc_ia(capsys, caplog): - testdata = [ - { - 'service': '--source https://web.archive.org/cdx/search/cdx', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, + testdata = ( + 'index https://web.archive.org/cdx/search/cdx --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': 'commoncrawl.org'}, - ] + ) # Disabled: minimize IA for ratelimit reasons - # [{'service': '--ia', 'mods': '--limit 10', 'cmd': 'size', 'rest': 'commoncrawl.org/*'}, - # {'count': 1, 'is_int': True}], - # [{'service': '--ia', 'mods': '--limit 10', 'cmd': 'size', 'rest': '--details commoncrawl.org/*'}, - # {'count': 2}], - # [{'service': '--ia', 'mods': '--from 20180101 --to 20180110 --limit 10', 'cmd': 'size', - # 'rest': '--details commoncrawl.org'}, - # {'count': 2}], + # ('index ia --limit 10 --url commoncrawl.org/* --count', {'count': 1, 'is_int': True}), + # ('index ia --from 20180101 --to 20180110 --limit 10 --url commoncrawl.org --count', {'count': 1}), multi_helper(testdata, capsys, caplog) -def warc_prefix_test_helper(tmpdir, prefix: str): - # crash testing only, so far - base = ' --limit 1 warc commoncrawl.org/*' +@conditional_mock_responses +def test_iter_alias(capsys): + # same requests as test_basics (`index cc --limit 10 --url commoncrawl.org/*`) + main(args='iter cc --limit 10 --url commoncrawl.org/*'.split()) + out, err = capsys.readouterr() + + assert len(out.splitlines()) == 10 + assert all(json.loads(line)['url'] for line in out.splitlines()) + assert '"iter" is deprecated' in err and 'use "cdxt index"' in err + + +@pytest.mark.parametrize( + 'args, expected', + [ + ('--crawl 2 size --details commoncrawl.org/*', "cdxt index cc --url 'commoncrawl.org/*' --crawl 2 --count"), + ( + '-v --ia --limit 1 warc --prefix FOO --url-fgrepv robots.txt commoncrawl.org/*', + "cdxt index ia --url 'commoncrawl.org/*' -v --limit 1 --url-fgrepv robots.txt" + " | cdxt repackage --prefix FOO --wb ia", + ), + ], +) +def test_removed_commands(capsys, args, expected): + with pytest.raises(SystemExit) as e: + main(args=args.split()) + assert e.value.code == 2 + + out, err = capsys.readouterr() + assert out == '' + assert 'has been removed' in err + assert expected in err + +def warc_test_helper(tmpdir, index_args: str, repackage_args: str = ''): with tmpdir.as_cwd(): - cmdline = prefix + base - if 'cc' in cmdline: - cmdline = cmdline.replace(' 1', ' 2') - print(cmdline, file=sys.stderr) - args = cmdline.split() - main(args=args) + for cmdline in ( + f'{index_args} -o captures.jsonl', + f'repackage captures.jsonl {repackage_args}', + ): + print(cmdline, file=sys.stderr) + main(args=cmdline.split()) @conditional_mock_responses def test_warc_prefix_1(tmpdir): - warc_prefix_test_helper(tmpdir, '-v -v --cc') + warc_test_helper(tmpdir, '-v -v index cc --limit 2 --url commoncrawl.org/*') + assert tmpdir.listdir(fil='REPACKAGE-001.warc.gz') @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_prefix_2(tmpdir): - warc_prefix_test_helper(tmpdir, '--ia') + warc_test_helper(tmpdir, 'index ia --limit 1 --url commoncrawl.org/*', '--wb ia') + assert tmpdir.listdir(fil='REPACKAGE-*.extracted.warc.gz') @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_prefix_3(tmpdir): - warc_prefix_test_helper(tmpdir, '--cc --cc-mirror https://index.commoncrawl.org/') + warc_test_helper(tmpdir, 'index cc --cc-mirror https://index.commoncrawl.org/ --limit 2 --url commoncrawl.org/*') + assert tmpdir.listdir(fil='REPACKAGE-001.warc.gz') @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_prefix_4(tmpdir): - warc_prefix_test_helper(tmpdir, '--source https://web.archive.org/cdx/search/cdx --wb https://web.archive.org/web') - - -def warc_suffix_test_helper(tmpdir, suffix: str): - # crash testing only, so far - base = ' --limit 1 warc commoncrawl.org/*' - prefix = '-v -v --cc' - - with tmpdir.as_cwd(): - cmdline = prefix + base + ' ' + suffix - print(cmdline, file=sys.stderr) - args = cmdline.split() - main(args=args) + warc_test_helper( + tmpdir, + 'index https://web.archive.org/cdx/search/cdx --limit 1 --url commoncrawl.org/*', + '--wb https://web.archive.org/web', + ) + assert tmpdir.listdir(fil='REPACKAGE-*.extracted.warc.gz') @conditional_mock_responses def test_warc_suffix_1(tmpdir): - warc_suffix_test_helper( + warc_test_helper( tmpdir, - '--prefix FOO --subprefix BAR --size 1 --creator creator --operator bob --url-fgrep common --url-fgrepv bar', + '-v -v index cc --limit 1 --url-fgrep common --url-fgrepv bar --url commoncrawl.org/*', + '--prefix FOO --subprefix BAR --size 1 --creator creator --operator bob', ) + assert tmpdir.listdir(fil='FOO-BAR-001.warc.gz') @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_suffix_2(tmpdir): - warc_suffix_test_helper(tmpdir, '--prefix EMPTY --size 1 --url-fgrep bar') + warc_test_helper( + tmpdir, '-v -v index cc --limit 1 --url-fgrep bar --url commoncrawl.org/*', '--prefix EMPTY --size 1' + ) + assert not tmpdir.listdir(fil='EMPTY-*') # no WARC file without records @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_suffix_3(tmpdir): - warc_suffix_test_helper(tmpdir, '--prefix EMPTY --size 1 --url-fgrepv common') + warc_test_helper( + tmpdir, '-v -v index cc --limit 1 --url-fgrepv common --url commoncrawl.org/*', '--prefix EMPTY --size 1' + ) + assert not tmpdir.listdir(fil='EMPTY-*') # no WARC file without records @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_suffix_4(tmpdir): - warc_suffix_test_helper( + warc_test_helper( tmpdir, - '--prefix FOO --subprefix BAR --size 1 --creator creator --operator bob --url-fgrep common --url-fgrepv bar', + '-v -v index cc --limit 1 --url-fgrep common --url-fgrepv bar --url commoncrawl.org/*', + '--prefix FOO --subprefix BAR --size 1 --creator creator --operator bob', ) + assert tmpdir.listdir(fil='FOO-BAR-001.warc.gz') -def one_ia_corner(tmpdir, cmdline): - with tmpdir.as_cwd(): - main(args=cmdline.split()) +def one_ia_corner(tmpdir, index_args): + warc_test_helper(tmpdir, 'index ia ' + index_args, '--wb ia') @pytest.mark.skip(reason='needs some ratelimit love') @@ -543,17 +426,15 @@ def test_warc_ia_corners(tmpdir, caplog): """ # revisit vivification - cmdline = '--ia --from 2017010118350 --to 2017010118350 warc pbm.com/robots.txt' - one_ia_corner(tmpdir, cmdline) + one_ia_corner(tmpdir, '--from 2017010118350 --to 2017010118350 --url pbm.com/robots.txt') # same-surt same-timestamp redir+200 - cmdline = '--ia --from 20090220001146 --to 20090220001146 warc pbm.com' - one_ia_corner(tmpdir, cmdline) + one_ia_corner(tmpdir, '--from 20090220001146 --to 20090220001146 --url pbm.com') # any redir -> 302, this is a 301 - cmdline = '--ia --from 2011020713024 --to 2011020713024 warc pbm.com' - one_ia_corner(tmpdir, cmdline) + one_ia_corner(tmpdir, '--from 2011020713024 --to 2011020713024 --url pbm.com') # warcing a 404 is a corner case in myrequests - cmdline = '--ia --from 20080512074145 --to 20080512074145 warc http://www.pbm.com/oly/archive/design94/0074.html' - one_ia_corner(tmpdir, cmdline) + one_ia_corner( + tmpdir, '--from 20080512074145 --to 20080512074145 --url http://www.pbm.com/oly/archive/design94/0074.html' + ) diff --git a/tests/unit/test_cc.py b/tests/unit/test_cc.py index b93f39d..47f1ac7 100644 --- a/tests/unit/test_cc.py +++ b/tests/unit/test_cc.py @@ -47,7 +47,7 @@ def test_apply_cc_defaults(): [{'crawl': 'foo'}, {}], [{'closest': '20180101'}, {'from_ts': '20171003000000', 'to': '20180401000000'}], [{'closest': '20180101', 'to': '20181201'}, {'from_ts': '20171003000000'}], - [{'to': '20180101'}, {'from_ts': '20170131235959'}], + [{'to': '20180101'}, {'from_ts': '20170101235959'}], # one year before the end of 2018-01-01 [{}, {'from_ts': '20170429003859'}], # uses now [{'from_ts': '20100101', 'closest': '20150301'}, {'to': '20150530000000'}], [{'from_ts': '20100101'}, {}], diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py new file mode 100644 index 0000000..6f79882 --- /dev/null +++ b/tests/unit/test_cdx_files.py @@ -0,0 +1,193 @@ +import gzip +import json +import shutil + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.indexes import IndexQuery, UsageError +from cdx_toolkit.indexes.cdx_files import ( + CdxFilesBackend, domain_entry_to_surts, load_url_patterns, url_pattern_to_surts, +) +from cdx_toolkit.record_io import write_records + +from tests.conftest import TEST_DATA_PATH + + +CDX_PATH = TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz' + +with gzip.open(CDX_PATH, 'rt') as _f: + LINES = _f.read().splitlines() + + +def count_lines(prefix, day=None): + return sum(1 for line in LINES if line.startswith(prefix) and (day is None or line.split(' ')[1].startswith(day))) + + +@pytest.mark.parametrize( + 'pattern, prefixes, exact', + [ + ('example.com/*', ['com,example)/'], []), + ('example.com/path*', ['com,example)/path'], []), + ('*.example.com', ['com,example)', 'com,example,'], []), + ('https://www.example.com/a', [], ['com,example)/a']), + ], +) +def test_url_pattern_to_surts(pattern, prefixes, exact): + assert url_pattern_to_surts(pattern) == (prefixes, exact) + + +@pytest.mark.parametrize('pattern', ['ex*ample.com', '*.example.com/path', '*.']) +def test_invalid_url_patterns(pattern): + with pytest.raises(UsageError): + url_pattern_to_surts(pattern) + + +def test_select_by_hostname_domain_and_url(): + backend = CdxFilesBackend(str(CDX_PATH)) + + assert backend.count(IndexQuery(hostnames=['biodiv.mnhn.fr'])) == count_lines('fr,mnhn,biodiv)') + assert backend.count(IndexQuery(domains=['missiondefrance.fr'])) == count_lines('fr,missiondefrance,') + assert backend.count(IndexQuery(domains=['mnhn.fr'], urls=['*.missiondefrance.fr'])) == len(LINES) + assert backend.count(IndexQuery(urls=['biodiv.mnhn.fr/fr/taxonomy*'])) == count_lines('fr,mnhn,biodiv)/fr/taxonomy') + + exact_key = LINES[0].split(' ')[0] + records = list(backend.iter_records(IndexQuery(urls=[json.loads(LINES[0].split(' ', 2)[2])['url']]))) + assert {r['urlkey'] for r in records} == {exact_key} + + +def test_time_range_fgrep_and_limit(): + backend = CdxFilesBackend(str(CDX_PATH)) + query = IndexQuery(domains=['missiondefrance.fr'], from_ts='20240716', to='20240716') + assert backend.count(query) == count_lines('fr,missiondefrance,', day='20240716') + + records = list(backend.iter_records(IndexQuery(domains=['mnhn.fr'], url_fgrep='/taxonomy/', limit=5))) + assert len(records) == 5 + assert all('/taxonomy/' in r['url'] and r['urlkey'].startswith('fr,mnhn,biodiv)') for r in records) + assert set(records[0]) >= {'urlkey', 'timestamp', 'url', 'filename', 'offset', 'length'} + + +def test_url_file_formats(tmp_path): + urls = ['*.missiondefrance.fr', 'biodiv.mnhn.fr/fr/taxonomy*'] + txt = tmp_path / 'urls.txt' + txt.write_text('# target URLs\n' + '\n'.join(urls) + '\n\n') + assert load_url_patterns(str(txt)) == urls + + write_records(({'name': str(i), 'target': u} for i, u in enumerate(urls)), str(tmp_path / 'urls.csv'), 'csv') + assert load_url_patterns(str(tmp_path / 'urls.csv'), 'target') == urls + with pytest.raises(UsageError, match="no column 'url'"): + load_url_patterns(str(tmp_path / 'urls.csv')) + with pytest.raises(UsageError, match='--url-column needs'): + load_url_patterns(str(txt), 'target') + + backend = CdxFilesBackend(str(CDX_PATH)) + expected = count_lines('fr,missiondefrance,') + count_lines('fr,mnhn,biodiv)/fr/taxonomy') + assert backend.count(IndexQuery(url_file=str(txt))) == expected + assert backend.count(IndexQuery(url_file=str(tmp_path / 'urls.csv'), url_column='target')) == expected + + +def test_url_file_parquet(tmp_path): + pytest.importorskip('pyarrow') + write_records(iter([{'url': '*.missiondefrance.fr'}]), str(tmp_path / 'urls.parquet'), 'parquet') + assert load_url_patterns(str(tmp_path / 'urls.parquet')) == ['*.missiondefrance.fr'] + + +@pytest.mark.parametrize('processes', [1, 2]) +def test_one_output_per_cdx_file(tmp_path, processes): + for name in ('a/cdx-00001.gz', 'b/cdx-00002.gz', 'b/cdx-00003.gz'): + (tmp_path / 'in' / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(CDX_PATH, tmp_path / 'in' / name) + # no match in this file: no output + with gzip.open(tmp_path / 'in' / 'b' / 'cdx-00003.gz', 'wt') as f: + f.write(next(line for line in LINES if line.startswith('fr,missiondefrance')) + '\n') + out = tmp_path / 'out' + + main([ + 'index', str(tmp_path / 'in' / '*' / 'cdx-*.gz'), '--hostname', 'biodiv.mnhn.fr', '-o', str(out), + '--processes', str(processes), + ]) + + outputs = sorted(str(p.relative_to(out)) for p in out.rglob('*') if p.is_file()) + assert outputs == ['a/cdx-00001.gz', 'b/cdx-00002.gz'] + with gzip.open(out / 'a' / 'cdx-00001.gz', 'rt') as f: + # the lines are copied unchanged + assert f.read().splitlines() == [line for line in LINES if line.startswith('fr,mnhn,biodiv)')] + + +def test_cli_combined_output_and_errors(tmp_path, capsys): + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '--limit', '3', '--fields', 'url', '--output-format', 'csv']) + out = capsys.readouterr().out.splitlines() + assert out[0] == 'url' and len(out) == 4 + + for args, error in [ + ([], 'select captures from the CDX files'), + (['--domain', 'mnhn.fr', '--filter', '=status:200'], '--filter cannot be used with the CDX files index'), + (['--domain', 'mnhn.fr', '--processes', '2'], '--processes can only be used'), + (['--domain', 'mnhn.fr', '-o', str(tmp_path / 'out'), '--limit', '1'], '--limit cannot be used'), + (['--domain', 'mnhn.fr', '-o', str(tmp_path / 'out'), '--fields', 'url'], '--fields cannot be used'), + (['--url-column', 'x', '--domain', 'mnhn.fr'], '--url-column and --url-file-type need --url-file'), + ]: + with pytest.raises(SystemExit): + main(['index', str(CDX_PATH)] + args) + assert error in capsys.readouterr().err, args + + +@pytest.mark.parametrize('index', ['cc', 'cc-url-index-s3']) +def test_url_file_only_for_cdx_files(capsys, index): + with pytest.raises(SystemExit): + main(['index', index, '--url-file', 'urls.txt']) + err = capsys.readouterr().err + assert '--url-file cannot be used with the' in err + if index != 'cc': + assert 'JOIN' in err + + +def test_dry_run(capsys): + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '--url', 'example.com/a', '--dry-run']) + out = capsys.readouterr().out + assert '1 CDX file(s)' in out and 'prefix fr,mnhn)' in out and 'exact com,example)/a' in out + + +@pytest.mark.parametrize( + 'entry, prefixes', + [ + ('example.com', ['com,example)', 'com,example,']), + ('https://example.com/', ['com,example)', 'com,example,']), + ('example.com/reports', ['com,example)/reports']), + ], +) +def test_domain_entry_to_surts(entry, prefixes): + assert domain_entry_to_surts(entry) == prefixes + + +def test_url_file_type_domain(): + """With --url-file-type domain, host entries match their domain with subdomains (like #73's filter_cdx).""" + url_list = str(TEST_DATA_PATH / 'repackage' / 'url_list.txt') + backend = CdxFilesBackend(str(CDX_PATH)) + + # e.g. 'bibliotheque.missiondefrance.fr' is a single URL as a pattern ... + assert backend.count(IndexQuery(url_file=url_list)) == 0 + # ... and a domain with --url-file-type domain; 'biodiv.mnhn.fr/fr/taxonomy' is a URL prefix + expected = count_lines('fr,missiondefrance,bibliotheque)') + count_lines('fr,mnhn,biodiv)/fr/taxonomy') + assert backend.count(IndexQuery(url_file=url_list, url_file_type='domain')) == expected == len(LINES) + + +@pytest.mark.parametrize( + 'args, error', + [ + (['--domain', 'mnhn.fr', '--url-file-type', 'domain'], '--url-column and --url-file-type need --url-file'), + (['--url-file', 'urls.txt', '--url-file-type', 'regex'], "invalid choice: 'regex'"), + ], +) +def test_url_file_type_errors(capsys, args, error): + with pytest.raises(SystemExit): + main(['index', str(CDX_PATH)] + args) + assert error in capsys.readouterr().err + + +def test_no_globs_over_http(capsys): + for command in (['index', 'https://example.org/indexes/cdx-*.gz', '--domain', 'mnhn.fr'], + ['repackage', 'https://example.org/indexes/cdx-*.gz']): + with pytest.raises(SystemExit): + main(command) + assert 'globs do not work with http(s) URLs' in capsys.readouterr().err diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index d8edc08..1b48ec1 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,8 +1,18 @@ -from argparse import Namespace +from argparse import ArgumentTypeError +import json from unittest.mock import Mock, patch -import cdx_toolkit.cli +import pytest + +from warcio.archiveiterator import ArchiveIterator + +import cdx_toolkit.cli.repackage +import cdx_toolkit.repackage.warc_utils import cdx_toolkit.warc +from cdx_toolkit.cli import main +from cdx_toolkit.cli.common import parse_size +from cdx_toolkit.cli.deprecated import get_command, rewrite_deprecated_command, translate +import cdx_toolkit.cli.index def _make_fake_record(): @@ -19,80 +29,218 @@ def _make_fake_record(): ) -def _build_warc_cmd(tmp_path, **overrides): - cmd = Namespace( - prefix=str(tmp_path / 'cli'), - subprefix=None, - url='http://example.com', - url_fgrep=None, - url_fgrepv=None, - creator=None, - operator=None, - ) - for key, value in overrides.items(): - setattr(cmd, key, value) - return cmd +def _write_captures(path, captures): + path.write_text(''.join(json.dumps(c) + '\n' for c in captures)) + return str(path) -def _make_capture(is_revisit=False, url='http://example.com'): - capture = Mock() - capture.__getitem__ = lambda self, key: {'url': url, 'timestamp': '20240101120000'}[key] - capture.is_revisit.return_value = is_revisit - capture.fetch_warc_record.return_value = _make_fake_record() - return capture +IA_CAPTURE = {'url': 'http://example.com', 'timestamp': '20240101120000', 'status': '200', 'mime': 'text/html'} +CC_CAPTURE = { + 'url': 'http://example.com', 'timestamp': '20240101120000', 'status': '200', + 'filename': 'crawl-data/x.warc.gz', 'offset': '10', 'length': '20', +} -def test_warcer_logs_revisit(tmp_path, caplog): - """warcer should log a warning when iterating over a revisit capture.""" - cmd = _build_warc_cmd(tmp_path) - fake_cdx = Mock() - fake_cdx.iter.return_value = iter([_make_capture(is_revisit=True)]) +def test_repackage_with_wb_logs_revisit(tmp_path, caplog): + captures = _write_captures(tmp_path / 'captures.jsonl', [dict(IA_CAPTURE, mime='warc/revisit')]) + prefix = str(tmp_path / 'out') - with patch.object(cdx_toolkit.cli, 'setup_cdx_fetcher_and_kwargs', return_value=(fake_cdx, {})): - with caplog.at_level('WARNING', logger='cdx_toolkit.cli'): - cdx_toolkit.cli.warcer(cmd, cmdline='test') + with patch.object(cdx_toolkit.cli.repackage, 'fetch_wb_warc', return_value=_make_fake_record()) as fetch: + with caplog.at_level('WARNING', logger='cdx_toolkit.cli.repackage'): + main(['repackage', captures, '--wb', 'ia', '--prefix', prefix]) + assert fetch.call_args.kwargs['wb'] == 'https://web.archive.org/web' assert 'revisit record being resolved' in caplog.text - assert list(tmp_path.glob('cli-*.warc.gz')) + assert list(tmp_path.glob('out-*.warc.gz')) + + +def _gzipped_warc_record(): + return cdx_toolkit.repackage.warc_utils.get_bytes_from_warc_record(_make_fake_record(), gzip=True) + + +def test_repackage_fetches_by_warc_location(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + resp = Mock(content=_gzipped_warc_record()) + + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=resp) as get: + main(['repackage', captures, '--prefix', str(tmp_path / 'out'), '--warc-download-prefix', 'https://x.org/']) + + assert get.call_args.args == ('https://x.org/crawl-data/x.warc.gz',) + assert get.call_args.kwargs == {'headers': {'Range': 'bytes=10-29'}} + + with open(tmp_path / 'out-001.warc.gz', 'rb') as f: + records = list(ArchiveIterator(f)) + assert [r.rec_type for r in records] == ['warcinfo', 'response'] + + +def test_repackage_without_warc_location_or_wb(tmp_path, capsys): + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE, IA_CAPTURE]) + + with pytest.raises(SystemExit) as e: + main(['repackage', captures, '--prefix', str(tmp_path / 'out')]) + + assert e.value.code == 2 + assert 'use --wb' in capsys.readouterr().err + assert not list(tmp_path.glob('out*')) # nothing is written + + +def test_repackage_without_captures_writes_no_file(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', []) + + main(['repackage', captures, '--prefix', str(tmp_path / 'out')]) + + assert not list(tmp_path.glob('out*')) + + +def test_repackage_default_prefix(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=Mock(content=_gzipped_warc_record())): + main(['repackage', captures]) -def test_warcer_skips_fgrep_and_fgrepv(tmp_path, caplog): - """warcer should skip URLs not matching --url-fgrep or matching --url-fgrepv.""" - cmd = _build_warc_cmd(tmp_path, url_fgrep='wanted', url_fgrepv='forbidden') - captures = [ - _make_capture(url='http://example.com/nope'), # fails fgrep - _make_capture(url='http://example.com/wanted/forbidden'), # fails fgrepv - _make_capture(url='http://example.com/wanted/keep'), # kept - ] - fake_cdx = Mock() - fake_cdx.iter.return_value = iter(captures) + assert [p.name for p in tmp_path.glob('*.warc.gz')] == ['REPACKAGE-001.warc.gz'] - with patch.object(cdx_toolkit.cli, 'setup_cdx_fetcher_and_kwargs', return_value=(fake_cdx, {})): - with caplog.at_level('DEBUG', logger='cdx_toolkit.cli'): - cdx_toolkit.cli.warcer(cmd, cmdline='test') - assert 'not warcing due to fgrep' in caplog.text - assert 'not warcing due to fgrepv' in caplog.text - assert captures[0].fetch_warc_record.call_count == 0 - assert captures[1].fetch_warc_record.call_count == 0 - assert captures[2].fetch_warc_record.call_count == 1 +def test_index_to_parquet_and_repackage(tmp_path, capsys): + pytest.importorskip('pyarrow') + captures = tmp_path / 'captures.parquet' + with patch.object(cdx_toolkit.cli.index, 'iter_index_records', return_value=iter([CC_CAPTURE])), \ + patch.object(cdx_toolkit.cli.index, 'resolve_index'): + main(['index', 'cc', '--url', 'example.com/*', '-o', str(captures)]) + with pytest.raises(SystemExit): + main(['index', 'cc', '--url', 'example.com/*', '--output-format', 'parquet']) + assert 'cannot be written to stdout' in capsys.readouterr().err -def test_warcer_passes_size_to_writer(tmp_path): - """warcer should pop 'size' out of kwargs and forward it to the writer.""" - cmd = _build_warc_cmd(tmp_path) - fake_cdx = Mock() - fake_cdx.iter.return_value = iter([]) - captured = {} + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=Mock(content=_gzipped_warc_record())): + main(['repackage', str(captures), '--prefix', str(tmp_path / 'out')]) + with open(tmp_path / 'out-001.warc.gz', 'rb') as f: + assert [r.rec_type for r in ArchiveIterator(f)] == ['warcinfo', 'response'] - real_get_writer = cdx_toolkit.warc.get_writer - def spy_get_writer(prefix, subprefix, info, **kwargs): - captured.update(kwargs) - return real_get_writer(prefix, subprefix, info, **kwargs) +def test_repackage_options_are_passed_to_the_warc_fetcher(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', []) + + with patch('cdx_toolkit.repackage.warc_filter.WARCFilter') as warc_filter: + main([ + 'repackage', captures, '--prefix', 'FOO', '--subprefix', 'BAR', '--size', '1MiB', + '--creator', 'me', '--description', 'test', '--add-metadata-record', 'a.txt', + '--parallel-readers', '4', '--warc-download-prefix', 's3://commoncrawl', + ]) + + kwargs = warc_filter.call_args.kwargs + assert kwargs['prefix_path'] == 'FOO' + assert kwargs['writer_subprefix'] == 'BAR' + assert kwargs['max_file_size'] == 1024 * 1024 + assert kwargs['write_paths_as_metadata_records'] == ['a.txt'] + assert kwargs['n_parallel_readers'] == 4 + assert kwargs['warc_download_prefix'] == 's3://commoncrawl' + info = kwargs['writer_info'] + assert info['isPartOf'] == 'FOO-BAR' + assert info['creator'] == 'me' + assert info['description'] == 'test' + + +def test_repackage_with_processes_uses_multiprocess(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + + with patch('cdx_toolkit.repackage.multiprocess.run_multiprocess_repackage', return_value=1) as run: + main(['repackage', captures, '--prefix', str(tmp_path / 'out'), '--processes', '3', '--subprefix', 'SUB']) + + kwargs = run.call_args.kwargs + assert (kwargs['n_processes'], kwargs['writer_subprefix']) == (3, 'SUB') + assert [job.filename for job in kwargs['source'].iter_range_jobs()] == ['crawl-data/x.warc.gz'] + + +def test_repackage_with_wb_rejects_fetcher_options(tmp_path, capsys): + captures = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE]) + + with pytest.raises(SystemExit): + main(['repackage', captures, '--wb', 'ia', '--processes', '2']) + assert '--processes cannot be used with --wb' in capsys.readouterr().err + + +def test_repackage_with_wb_passes_size_and_warcinfo_to_writer(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', []) + + with patch.object(cdx_toolkit.warc, 'get_writer', wraps=cdx_toolkit.warc.get_writer) as get_writer: + main([ + 'repackage', captures, '--wb', 'ia', '--prefix', 'FOO', '--subprefix', 'BAR', '--size', '1MiB', + '--creator', 'me', '--description', 'test', + ]) + + prefix, subprefix, info = get_writer.call_args.args + assert (prefix, subprefix) == ('FOO', 'BAR') + assert get_writer.call_args.kwargs == {'size': 1024 * 1024} + assert info['isPartOf'] == 'FOO-BAR' + assert info['creator'] == 'me' + assert info['description'] == 'test' + + +@pytest.mark.parametrize( + 'value, expected', + [('1000', 1000), ('1kB', 1000), ('100MB', 100 * 1000 ** 2), ('1.5GB', 1500 * 1000 ** 2), ('1GiB', 1024 ** 3)], +) +def test_parse_size(value, expected): + assert parse_size(value) == expected + + +@pytest.mark.parametrize('value', ['', 'GB', '1XB', '-1']) +def test_parse_size_invalid(value): + with pytest.raises(ArgumentTypeError): + parse_size(value) + + +@pytest.mark.parametrize( + 'args, command', + [ + ('--cc --limit 10 iter example.com', 'iter'), + ('--source https://example.com/cdx --crawl 3 size example.com', 'size'), + ('-v index cc --url example.com', 'index'), + ('--version', None), + ], +) +def test_get_command(args, command): + assert get_command(args.split()) == command + + +@pytest.mark.parametrize( + 'args, expected', + [ + ('iter cc --url example.com/*', ['index', 'cc', '--url', 'example.com/*']), + ('-v iter --help', ['-v', 'index', '--help']), + # the old syntax is not translated, the arguments are the same as for `index` + ('--cc --limit 10 iter example.com/*', ['--cc', '--limit', '10', 'index', 'example.com/*']), + ], +) +def test_iter_is_an_alias_of_index(capsys, args, expected): + assert rewrite_deprecated_command(args.split()) == expected + assert '"iter" is deprecated' in capsys.readouterr().err + + +def test_iter_runs_like_index(capsys): + with patch.object(cdx_toolkit.cli.index, 'run') as run: + main(['iter', 'cc', '--url', 'example.com/*', '--limit', '3', '-o', 'out.csv']) + + args = run.call_args.args[0] + assert (args.cmd, args.index, args.urls) == ('index', 'cc', ['example.com/*']) + assert (args.limit, args.output) == (3, 'out.csv') + assert '"iter" is deprecated' in capsys.readouterr().err + + +def test_translate_warc(): + args = '--source https://example.com/cdx --wb https://example.com/wb --limit=5 warc --size 1000 example.com/*' + assert translate(args.split()) == ( + "cdxt index https://example.com/cdx --url 'example.com/*' --limit 5" + " | cdxt repackage --wb https://example.com/wb --size 1000" + ) - with patch.object(cdx_toolkit.cli, 'setup_cdx_fetcher_and_kwargs', return_value=(fake_cdx, {'size': 42})): - with patch.object(cdx_toolkit.warc, 'get_writer', side_effect=spy_get_writer): - cdx_toolkit.cli.warcer(cmd, cmdline='test') - assert captured == {'size': 42} +def test_verbose_before_and_after_subcommand(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', []) + with patch.object(cdx_toolkit.cli, 'set_loglevel') as set_loglevel: + main(['-v', 'repackage', captures, '--prefix', str(tmp_path / 'out')]) + assert set_loglevel.call_args.args == (1,) + main(['repackage', '-vv', captures, '--prefix', str(tmp_path / 'out')]) + assert set_loglevel.call_args.args == (2,) diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py new file mode 100644 index 0000000..366a049 --- /dev/null +++ b/tests/unit/test_indexes.py @@ -0,0 +1,131 @@ +from unittest.mock import patch + +import pytest + +from cdx_toolkit.indexes import IndexQuery, UsageError, iter_index_records, resolve_index +from cdx_toolkit.indexes.cdx_api import CdxApiBackend +from cdx_toolkit.indexes.cdx_files import CdxFilesBackend +from cdx_toolkit.indexes.resolve import is_cdx_api_location + + +@pytest.mark.parametrize( + 'location, is_api', + [ + ('cc', True), + ('ia', True), + ('http://index.commoncrawl.org', True), + ('https://index.commoncrawl.org/', True), + ('https://index.commoncrawl.org/CC-MAIN-2024-30-index', True), + ('https://web.archive.org/cdx/search/cdx', True), + ('https://data.commoncrawl.org/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-00000.gz', False), + ('https://example.com/index.cdxj', False), + ('https://example.com/indexes/*', False), + ('s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/*.cdx.gz', False), + ('./captures.cdx', False), + ('/data/index', False), + ], +) +def test_is_cdx_api_location(location, is_api): + assert is_cdx_api_location(location) == is_api + + +def test_resolve_index(): + backend = resolve_index('cc', cc_mirror='https://mirror.example.com/', cc_sort='ascending') + assert isinstance(backend, CdxApiBackend) + assert (backend.source, backend.cc_mirror, backend.cc_sort) == ('cc', 'https://mirror.example.com/', 'ascending') + + cdx_files = resolve_index('s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz') + assert isinstance(cdx_files, CdxFilesBackend) + with pytest.raises(UsageError, match='unknown index format'): + resolve_index('cc', index_format='parquet') + + +def test_query_options_used(): + query = IndexQuery(urls=['example.com/*'], limit=10) + assert query.options_used() == {'urls', 'limit'} + assert IndexQuery().options_used() == set() + + +def test_cdx_api_validate(): + CdxApiBackend('cc').validate(IndexQuery(domains=['example.com'], crawl='3')) + + with pytest.raises(UsageError, match='at least one --url'): + CdxApiBackend('cc').validate(IndexQuery(crawl='3')) + with pytest.raises(UsageError, match='--crawl can only be used'): + CdxApiBackend('ia').validate(IndexQuery(urls=['example.com'], crawl='3')) + + +def test_unsupported_option(): + class NoFilterBackend(CdxApiBackend): + supported_options = CdxApiBackend.supported_options - {'filters'} + + with pytest.raises(UsageError, match='--filter cannot be used with the CDX API index'): + NoFilterBackend('cc').validate(IndexQuery(urls=['example.com'], filters=['=status:200'])) + + +def _capture(url): + return {'url': url, 'timestamp': '20240101000000'} + + +def test_cdx_api_url_patterns_and_limit(): + query = IndexQuery( + urls=['example.com/*'], hostnames=['www.example.org'], domains=['example.net'], + from_ts='2020', filters=['=status:200'], limit=3, + ) + results = { + 'example.com/*': [_capture('http://example.com/a'), _capture('http://example.com/b')], + 'www.example.org': [_capture('http://www.example.org/'), _capture('http://www.example.org/x')], + 'example.net': [_capture('http://example.net/')], + } + + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.iter.side_effect = lambda url, **params: iter(results[url]) + records = list(CdxApiBackend('cc').iter_records(query)) + + assert [r['url'] for r in records] == ['http://example.com/a', 'http://example.com/b', 'http://www.example.org/'] + + calls = fetcher.return_value.iter.call_args_list + assert len(calls) == 2 # the limit was reached before the domain query + assert calls[0].args == ('example.com/*',) + assert calls[0].kwargs == {'limit': 3, 'from_ts': '2020', 'filter': ['=status:200']} + assert calls[1].args == ('www.example.org',) + assert calls[1].kwargs == {'limit': 1, 'from_ts': '2020', 'filter': ['=status:200'], 'matchType': 'host'} + + +def test_cdx_api_closest_uses_get_and_crawl_is_normalized(): + query = IndexQuery(domains=['example.com'], closest='20170615', crawl='CC-MAIN-2024-33,CC-MAIN-2024-30') + + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.get.return_value = [_capture('http://example.com/')] + records = list(CdxApiBackend('cc').iter_records(query)) + + assert len(records) == 1 + assert fetcher.call_args.kwargs == {'source': 'cc', 'crawl': ['CC-MAIN-2024-33', 'CC-MAIN-2024-30']} + assert fetcher.return_value.get.call_args.kwargs == {'closest': '20170615', 'matchType': 'domain'} + + +def test_cdx_api_count(): + query = IndexQuery(urls=['example.com/*'], domains=['example.org'], limit=10) + + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.get_size_estimate.side_effect = [3000, 1500] + assert CdxApiBackend('ia').count(query) == 4500 + + calls = fetcher.return_value.get_size_estimate.call_args_list + assert calls[1].args == ('example.org',) + assert calls[1].kwargs == {'limit': 10, 'matchType': 'domain'} + + +def test_iter_index_records_url_fgrep(): + backend = CdxApiBackend('cc') + query = IndexQuery(urls=['example.com/*'], url_fgrep='/keep', url_fgrepv='robots.txt') + captures = [ + _capture('http://example.com/drop'), + _capture('http://example.com/keep/robots.txt'), + _capture('http://example.com/keep/this'), + ] + + with patch.object(backend, 'iter_records', return_value=iter(captures)): + records = list(iter_index_records(backend, query)) + + assert [r['url'] for r in records] == ['http://example.com/keep/this'] diff --git a/tests/unit/test_record_io.py b/tests/unit/test_record_io.py new file mode 100644 index 0000000..a88ebe6 --- /dev/null +++ b/tests/unit/test_record_io.py @@ -0,0 +1,146 @@ +import io +import json + +import pytest + +from cdx_toolkit.record_io import RecordWriter, format_from_path, is_glob, read_records, sniff_format, write_records + + +RECORDS = [ + {'urlkey': 'com,example)/', 'timestamp': '20240101000000', 'url': 'http://example.com/', 'status': '200', + 'filename': 'crawl-data/a.warc.gz', 'offset': '10', 'length': '20'}, + {'urlkey': 'com,example)/x', 'timestamp': '20240102000000', 'url': 'http://example.com/x', 'status': '404', + 'filename': 'crawl-data/b.warc.gz', 'offset': '30', 'length': '40'}, +] + + +@pytest.mark.parametrize( + 'path, fmt', + [ + ('-', None), + ('out.jsonl', 'jsonl'), + ('out.json.gz', 'jsonl'), + ('s3://bucket/out.cdxj', 'cdxj'), + ('out.cdx.gz', 'cdxj'), + ('OUT.CSV', 'csv'), + ('out.tsv.gz', 'tsv'), + ('out.txt', None), + ], +) +def test_format_from_path(path, fmt): + assert format_from_path(path) == fmt + + +@pytest.mark.parametrize('fmt', ['jsonl', 'cdxj', 'csv', 'tsv']) +@pytest.mark.parametrize('extension', ['', '.gz']) +def test_round_trip(tmp_path, fmt, extension): + path = str(tmp_path / f'records.{fmt}{extension}') + assert write_records(iter(RECORDS), path, fmt) == 2 + + # the format is detected from the content, not the extension + assert list(read_records(path)) == RECORDS + + +def test_writer_fields(): + fh = io.StringIO() + writer = RecordWriter(fh, 'csv', fields=['url', 'status', 'missing']) + for record in RECORDS: + writer.write(record) + assert fh.getvalue().splitlines() == ['url,status,missing', 'http://example.com/,200,', 'http://example.com/x,404,'] + + fh = io.StringIO() + RecordWriter(fh, 'cdxj', fields=['url']).write(RECORDS[0]) + assert fh.getvalue() == 'com,example)/ 20240101000000 {"url": "http://example.com/"}\n' + + +def test_writer_cdxj_requires_urlkey(): + with pytest.raises(ValueError, match='urlkey'): + RecordWriter(io.StringIO(), 'cdxj').write({'url': 'http://example.com/', 'timestamp': '2024'}) + + +def test_read_columnar_index_names(tmp_path): + path = tmp_path / 'range-jobs.csv' + path.write_text('url,warc_filename,warc_record_offset,warc_record_length\nhttp://example.com/,a.warc.gz,1,2\n') + assert list(read_records(str(path))) == [ + {'url': 'http://example.com/', 'filename': 'a.warc.gz', 'offset': '1', 'length': '2'} + ] + + +def test_read_empty_and_stdin(tmp_path, monkeypatch): + path = tmp_path / 'empty.jsonl' + path.write_text('') + assert list(read_records(str(path))) == [] + + monkeypatch.setattr('sys.stdin', io.StringIO(json.dumps(RECORDS[0]) + '\n')) + assert list(read_records('-')) == [RECORDS[0]] + + +@pytest.mark.parametrize( + 'line, fmt', + [ + ('{"url": "http://example.com/"}', 'jsonl'), + ('com,example)/ 20240101000000 {"url": "http://example.com/"}', 'cdxj'), + ('url\tfilename\toffset\tlength', 'tsv'), + ('url,filename,offset,length', 'csv'), + ], +) +def test_sniff_format(line, fmt): + assert sniff_format(line) == fmt + + +def test_parquet_round_trip(tmp_path): + pytest.importorskip('pyarrow') + path = str(tmp_path / 'records.parquet') + assert format_from_path(path) == 'parquet' + assert write_records(iter(RECORDS), path, 'parquet') == 2 + assert list(read_records(path)) == RECORDS + + # selected fields, missing values are left out when reading + write_records(iter(RECORDS), path, 'parquet', fields=['url', 'status', 'missing']) + assert list(read_records(path)) == [ + {'url': 'http://example.com/', 'status': '200'}, {'url': 'http://example.com/x', 'status': '404'}, + ] + + +def test_parquet_many_batches(tmp_path, monkeypatch): + pytest.importorskip('pyarrow') + import pyarrow.parquet + + monkeypatch.setattr('cdx_toolkit.record_io.PARQUET_BATCH_SIZE', 3) + records = [{'url': f'http://example.com/{i}', 'offset': i} for i in range(7)] + path = str(tmp_path / 'records.parquet') + + assert write_records(iter(records), path, 'parquet') == 7 + assert pyarrow.parquet.ParquetFile(path).metadata.num_row_groups == 3 + assert [r['offset'] for r in read_records(path)] == [str(i) for i in range(7)] # all columns are strings + + +def test_parquet_needs_a_file(tmp_path): + with pytest.raises(ValueError, match='stdout'): + write_records(iter(RECORDS), '-', 'parquet') + + +def test_cdx_is_an_alias_of_cdxj(tmp_path): + path = str(tmp_path / 'records.cdx') + assert format_from_path(path) == 'cdxj' + write_records(iter(RECORDS), path, 'cdx') + assert open(path).read().startswith('com,example)/ 20240101000000 {') + assert list(read_records(path, 'cdx')) == RECORDS + + +@pytest.mark.parametrize( + 'path, glob', + [ + ('https://example.org/index.cdx.gz?signature=x', False), + ('https://example.org/indexes/*.gz', True), + ('s3://bucket/cdx-0000?.gz', True), + ('-', False), + ], +) +def test_is_glob(path, glob): + assert is_glob(path) == glob + + +def test_no_globs_over_http(): + with pytest.raises(ValueError, match='globs do not work with http'): + list(read_records('https://data.commoncrawl.org/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz')) diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py new file mode 100644 index 0000000..0908262 --- /dev/null +++ b/tests/unit/test_sql_indexes.py @@ -0,0 +1,305 @@ +import datetime +import json +from unittest.mock import patch + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.indexes import IndexQuery, UsageError, resolve_index +from cdx_toolkit.indexes.cdx_api import CdxApiBackend +from cdx_toolkit.indexes.cdx_files import CdxFilesBackend +from cdx_toolkit.indexes.sql import AthenaBackend, check_language, DuckDbBackend, normalize_row, parse_athena_location + +from tests.conftest import requires_duckdb + + +ATHENA = 'athena://athena.us-east-1.amazonaws.com:443/ccindex?S3OutputLocation=s3://my-bucket/athena/' + + +@pytest.mark.parametrize( + 'location, index_format, backend, backend_location', + [ + ('cc', 'auto', CdxApiBackend, None), + ('cc-url-index-s3', 'auto', DuckDbBackend, 's3://commoncrawl/cc-index/table/cc-main/warc/'), + ('cc-url-index-hf', 'auto', DuckDbBackend, 'hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/'), + ('./index/part-0.parquet', 'auto', DuckDbBackend, './index/part-0.parquet'), + ('s3://bucket/index/*.parquet', 'auto', DuckDbBackend, 's3://bucket/index/*.parquet'), + ('s3://bucket/index/', 'duckdb', DuckDbBackend, 's3://bucket/index/'), + (ATHENA, 'auto', AthenaBackend, ATHENA), + ('s3://bucket/indexes/cdx-00000.gz', 'auto', CdxFilesBackend, 's3://bucket/indexes/cdx-00000.gz'), + ('./indexes/*.cdx', 'cdx', CdxFilesBackend, './indexes/*.cdx'), + ], +) +def test_resolve_index(location, index_format, backend, backend_location): + resolved = resolve_index(location, index_format) + assert isinstance(resolved, backend) + if backend_location: + assert resolved.location == backend_location + + +@pytest.mark.parametrize( + 'location, index_format, error', + [ + ('cc', 'duckdb', 'cannot be queried with DuckDB'), + ('s3://bucket/index/', 'athena', 'invalid Athena location'), + ('athena://athena.us-east-1.amazonaws.com/ccindex', 'auto', 'S3OutputLocation'), + ], +) +def test_resolve_index_errors(location, index_format, error): + with pytest.raises(UsageError, match=error): + resolve_index(location, index_format) + + +def test_parse_athena_location(): + assert parse_athena_location(ATHENA) == ('us-east-1', 'ccindex', 'ccindex', 's3://my-bucket/athena/') + assert parse_athena_location( + 'athena://athena.eu-west-1.amazonaws.com/mydb?S3OutputLocation=s3://b/p/&table=urls' + ) == ('eu-west-1', 'mydb', 'urls', 's3://b/p/') + + +def test_normalize_row(): + row = { + 'url': 'https://example.com/', + 'fetch_time': datetime.datetime(2024, 7, 13, 10, 22, 33), + 'fetch_status': 200, + 'warc_record_offset': 10, + 'content_languages': None, + 'my_column': 'x', + } + assert normalize_row(row) == { + 'url': 'https://example.com/', 'timestamp': '20240713102233', 'status': '200', 'offset': '10', 'my_column': 'x', + } + # Athena returns strings + assert normalize_row({'fetch_time': '2024-07-13 10:22:33.000'}) == {'timestamp': '20240713102233'} + + +@pytest.mark.parametrize( + 'query, error', + [ + (IndexQuery(urls=['ex*ample.com']), 'invalid URL pattern'), + (IndexQuery(sql='SELECT 1', urls=['example.com/*']), '--url cannot be combined with --sql'), + (IndexQuery(domains=['example.com'], filters=['=status:200']), '--filter cannot be used'), + (IndexQuery(url_file='urls.csv'), 'use a custom --sql query that joins the index with a table'), + (IndexQuery(domains=['example.com'], languages=['german']), 'invalid language'), + (IndexQuery(domains=['example.com'], languages=['en']), 'use ISO 639-3 codes'), + (IndexQuery(domains=['example.com'], to='2024-01'), 'invalid timestamp'), + (IndexQuery(sql='SELECT 1', from_ts='2024'), '--from cannot be combined with --sql'), + (IndexQuery(), 'needs --url, --hostname, --domain or --sql'), + ( + IndexQuery(sql='SELECT 1', domains=['example.com'], limit=1), + '--domain, --limit cannot be combined with --sql', + ), + ], +) +def test_sql_validate(query, error): + with pytest.raises(UsageError, match=error): + DuckDbBackend('s3://bucket/index/').validate(query) + + +def test_duckdb_guided_query_prunes_crawls(): + backend = DuckDbBackend('s3://bucket/index/') + query = IndexQuery(hostnames=['www.example.com'], domains=['example.org'], crawl='CC-MAIN-2024-33', limit=5) + + with patch('cdx_toolkit.indexes.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): + sql, n_crawls = backend.prepare(query) + + assert n_crawls == 1 + assert "read_parquet(['s3://bucket/index/crawl=CC-MAIN-2024-33/subset=warc/*.parquet']" in sql + assert 'crawl IN' not in sql # pruned by the glob + assert "url_host_name = 'www.example.com' OR url_host_registered_domain = 'example.org'" in sql + assert 'url_surtkey, fetch_time, url,' in sql + assert sql.rstrip().endswith('LIMIT 5') + + +def test_guided_query_url_patterns(): + query = IndexQuery(urls=['example.com/a_b*', 'https://www.example.org/', '*.example.net'], domains=['example.com']) + sql, _ = AthenaBackend(ATHENA).prepare(query) + + assert "(url_host_tld = 'com' OR url_host_tld = 'net' OR url_host_tld = 'org')" in sql + assert ("(url_host_registered_domain = 'example.com' OR url_surtkey = 'org,example)/' " + "OR url_surtkey LIKE 'com,example)/a\\_b%' ESCAPE '\\' " + "OR url_surtkey LIKE 'net,example)%' ESCAPE '\\' OR url_surtkey LIKE 'net,example,%' ESCAPE '\\')") in sql + + +def test_guided_query_time_range_and_languages(): + query = IndexQuery(hostnames=['example.com'], from_ts='2024', to='20240410', languages=['deu', 'fra']) + sql, _ = AthenaBackend(ATHENA).prepare(query) + + assert "fetch_time >= TIMESTAMP '2024-01-01 00:00:00'" in sql + assert "fetch_time <= TIMESTAMP '2024-04-10 23:59:59'" in sql + assert ("(content_languages = 'deu' OR content_languages LIKE 'deu,%' " + "OR content_languages = 'fra' OR content_languages LIKE 'fra,%')") in sql + + +def test_athena_guided_query_and_cost_guard(): + backend = AthenaBackend(ATHENA) + query = IndexQuery(domains=['example.com']) + + sql, n_crawls = backend.prepare(query) + assert n_crawls is None + assert 'FROM ccindex' in sql + + # not restricted to crawls -> confirmation needed, refused in non-interactive mode + with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch.object(backend, 'iter_sql_rows') as rows: + stdin.isatty.return_value = False + with pytest.raises(SystemExit, match='--yes'): + list(backend.iter_records(query)) + rows.assert_not_called() + + backend.assume_yes = True + with patch.object(backend, 'iter_sql_rows', return_value=iter([{'url': 'https://example.com/'}])) as rows: + assert list(backend.iter_records(query)) == [{'url': 'https://example.com/'}] + assert rows.call_args.args == (sql,) + + +def test_sql_count_and_explain(): + backend = AthenaBackend(ATHENA, assume_yes=True) + query = IndexQuery(sql='SELECT url FROM ccindex WHERE crawl = \'CC-MAIN-2024-33\';\n') + + with patch.object(backend, 'iter_sql_rows', return_value=iter([{'n': '42'}])) as rows: + assert backend.count(query) == 42 + assert rows.call_args.args == ( + "SELECT count(*) AS n FROM (SELECT url FROM ccindex WHERE crawl = 'CC-MAIN-2024-33') AS q", + ) + + assert 'SELECT url FROM ccindex' in backend.explain(query) + + +@pytest.fixture +def parquet_index(tmp_path): + """A tiny columnar index, partitioned like Common Crawl's (crawl=.../subset=warc/*.parquet).""" + import duckdb + + base = tmp_path / 'index' + rows = [ + ('CC-MAIN-2024-33', 'org,example)/', 'https://example.org/', 'example.org', 'www.example.org', 'a.warc.gz', 30), + ('CC-MAIN-2024-33', 'com,example)/', 'https://example.com/', 'example.com', 'example.com', 'b.warc.gz', 20), + ('CC-MAIN-2024-33', 'com,example)/x', 'https://example.com/x', 'example.com', 'example.com', 'a.warc.gz', 10), + ('CC-MAIN-2024-30', 'com,example)/', 'https://example.com/', 'example.com', 'example.com', 'c.warc.gz', 0), + ] + con = duckdb.connect() + for crawl in ('CC-MAIN-2024-30', 'CC-MAIN-2024-33'): + path = base / f'crawl={crawl}' / 'subset=warc' + path.mkdir(parents=True) + values = ', '.join( + f"('{r[1]}', TIMESTAMP '2024-08-01 12:00:00', '{r[2]}', 'text/html', 200, '{r[3]}', '{r[4]}', " + f"'{r[3].split('.')[-1]}', '{r[5]}', {r[6]}, 100)" + for r in rows if r[0] == crawl + ) + con.execute(f''' + COPY (SELECT *, NULL::VARCHAR AS content_mime_detected, NULL::VARCHAR AS content_digest, + NULL::VARCHAR AS content_languages, NULL::VARCHAR AS content_charset, + NULL::VARCHAR AS fetch_redirect + FROM (VALUES {values}) AS t(url_surtkey, fetch_time, url, content_mime_type, + fetch_status, url_host_registered_domain, url_host_name, url_host_tld, warc_filename, + warc_record_offset, warc_record_length)) + TO '{path}/part-0.parquet' (FORMAT parquet) + ''') + return base + + +@requires_duckdb +def test_index_duckdb_end_to_end(parquet_index, tmp_path, capsys): + output = tmp_path / 'out.jsonl' + with patch('cdx_toolkit.indexes.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): + main([ + 'index', str(parquet_index), '--index-format', 'duckdb', '--domain', 'example.com', + '--crawl', 'CC-MAIN-2024-33', '-o', str(output), + ]) + + records = [json.loads(line) for line in output.read_text().splitlines()] + # sorted by WARC file and offset + assert [(r['filename'], r['offset']) for r in records] == [('a.warc.gz', '10'), ('b.warc.gz', '20')] + assert records[0] == { + 'urlkey': 'com,example)/x', 'timestamp': '20240801120000', 'url': 'https://example.com/x', 'mime': 'text/html', + 'status': '200', 'filename': 'a.warc.gz', 'offset': '10', 'length': '100', 'crawl': 'CC-MAIN-2024-33', + } + + +@requires_duckdb +def test_index_duckdb_parquet_glob_sql_and_count(parquet_index, tmp_path, capsys, monkeypatch): + files = f'{parquet_index}/*/*/*.parquet' + + # all crawls -> needs --yes in non-interactive mode + main(['index', files, '--hostname', 'www.example.org', '--yes', '--output-format', 'csv', '--fields', 'url,crawl']) + assert capsys.readouterr().out.splitlines() == ['url,crawl', 'https://example.org/,CC-MAIN-2024-33'] + + main(['index', files, '--domain', 'example.com', '--count', '--yes']) + assert capsys.readouterr().out == '3\n' + + sql = tmp_path / 'query.sql' + sql.write_text(f"SELECT url, warc_filename FROM read_parquet('{files}') WHERE warc_record_offset > 15;") + main(['index', files, '--sql', str(sql), '--yes', '--output-format', 'csv']) + assert sorted(capsys.readouterr().out.splitlines()) == [ + 'https://example.com/,b.warc.gz', 'https://example.org/,a.warc.gz', 'url,filename' + ] + + main(['index', files, '--sql', str(sql), '--dry-run']) + assert 'WHERE warc_record_offset > 15' in capsys.readouterr().out + + +@requires_duckdb +def test_index_duckdb_time_range_and_language(tmp_path, capsys): + import duckdb + + path = tmp_path / 'crawl=CC-MAIN-2024-33' / 'subset=warc' + path.mkdir(parents=True) + rows = [ + # url, fetch_time (UTC), content_languages + ('https://example.com/a', '2024-08-01 23:30:00', 'deu'), + ('https://example.com/b', '2024-08-02 00:30:00', 'eng,deu'), + ('https://example.com/c', '2024-08-03 12:00:00', 'deu,eng'), + ('https://example.com/d', '2024-08-04 12:00:00', None), + ] + values = ', '.join( + f"('{url}', TIMESTAMPTZ '{t}+00', {repr(lang) if lang else 'NULL'}, 'example.com', 'example.com', 'com', " + f"'a.warc.gz', {i}, 100)" + for i, (url, t, lang) in enumerate(rows) + ) + duckdb.connect().execute(f''' + COPY (SELECT *, NULL::VARCHAR AS url_surtkey, NULL::VARCHAR AS content_mime_type, + NULL::VARCHAR AS content_mime_detected, NULL::SMALLINT AS fetch_status, + NULL::VARCHAR AS content_digest, NULL::VARCHAR AS content_charset, NULL::VARCHAR AS fetch_redirect + FROM (VALUES {values}) AS t(url, fetch_time, content_languages, url_host_registered_domain, + url_host_name, url_host_tld, warc_filename, warc_record_offset, warc_record_length)) + TO '{path}/part-0.parquet' (FORMAT parquet) + ''') + files = f'{tmp_path}/*/*/*.parquet' + + def urls(*args): + main(['index', files, '--domain', 'example.com', '--yes', '--output-format', 'csv', '--fields', 'url,timestamp'] + + list(args)) + return capsys.readouterr().out.splitlines()[1:] + + # timestamps are UTC, the day of --to is included + assert urls('--from', '20240802', '--to', '20240803') == [ + 'https://example.com/b,20240802003000', 'https://example.com/c,20240803120000', + ] + # the primary (first) content language + assert urls('--language', 'deu') == ['https://example.com/a,20240801233000', 'https://example.com/c,20240803120000'] + assert urls('--language', 'eng', '--language', 'deu', '--to', '2024080123') == [ + 'https://example.com/a,20240801233000', + ] + + +@pytest.mark.parametrize('code, language', [('eng', 'eng'), ('DEU', 'deu'), (' fra ', 'fra'), ('nds', 'nds')]) +def test_check_language(code, language): + assert check_language(code) == language + + +def test_languages_are_normalized_in_the_query(): + sql, _ = AthenaBackend(ATHENA).prepare(IndexQuery(hostnames=['example.com'], languages=['DEU'])) + assert "content_languages = 'deu'" in sql + + +@requires_duckdb +def test_index_duckdb_url_patterns(parquet_index, capsys): + files = f'{parquet_index}/*/*/*.parquet' + + def urls(*args): + main(['index', files, '--yes', '--output-format', 'csv', '--fields', 'url'] + list(args)) + return sorted(capsys.readouterr().out.splitlines()[1:]) + + assert urls('--url', 'example.com/x') == ['https://example.com/x'] + assert urls('--url', 'example.com/*') == ['https://example.com/', 'https://example.com/', 'https://example.com/x'] + assert urls('--url', '*.example.org', '--url', 'example.com/x') == ['https://example.com/x', 'https://example.org/'] diff --git a/tests/unit/test_timeutils.py b/tests/unit/test_timeutils.py index 7f090d8..86f6f6c 100644 --- a/tests/unit/test_timeutils.py +++ b/tests/unit/test_timeutils.py @@ -7,6 +7,9 @@ def test_padding(): assert timeutils.pad_timestamp('1998') == '19980101000000' assert timeutils.pad_timestamp_up('199802') == '19980228235959' assert timeutils.pad_timestamp_up('199812') == '19981231235959' + assert timeutils.pad_timestamp_up('202402') == '20240229235959' # leap year + assert timeutils.pad_timestamp_up('20260410') == '20260410235959' # the day is kept + assert timeutils.pad_timestamp_up('2026041012') == '20260410125959' def test_time_timestamp(): From 45dbc85ac73f017882b95a2eecfe3f76ba454c56 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 8 Oct 2026 12:45:34 +0200 Subject: [PATCH 02/91] docs: document the new CLI and add a migration guide - README: `cdxt index` (index types, selection, output, SQL and CDX file options) and `cdxt repackage`, new extras, migration table - docs/cli-migration-guide.md: old `iter`/`size`/`warc` commands and the feat/warc-range-sources `repackage` options mapped to the new CLI, with examples - docs/notes/warc-fetcher-performance.md: WARC fetcher benchmarks and tuning notes from feat/warc-range-sources - CHANGELOG entry --- CHANGELOG.md | 29 +++ README.md | 202 +++++++++++++++-- docs/cli-migration-guide.md | 303 +++++++++++++++++++++++++ docs/notes/warc-fetcher-performance.md | 201 ++++++++++++++++ 4 files changed, 710 insertions(+), 25 deletions(-) create mode 100644 docs/cli-migration-guide.md create mode 100644 docs/notes/warc-fetcher-performance.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cfe21c..9a31d2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,32 @@ +- unreleased + + new command line interface: `cdxt index` and `cdxt repackage` (see README) + + `cdxt iter` is a deprecated alias of `cdxt index` (same arguments, prints a deprecation warning) + + removed `cdxt size` and `cdxt warc`; they print the equivalent new command + + `cdxt index`: `--hostname`, `--domain`, repeatable `--url`, `-o/--output` (local or remote), + JSONL/CDXJ/CSV/TSV output, all fields by default, `--count` replaces `size` + + `cdxt repackage`: reads captures from a file or stdin, human-readable `--size`, `--is-part-of`, `--description` + + `--cc-sort` is now passed on to the CDX fetcher + + `cdxt index` queries Common Crawl's columnar URL index with DuckDB (`cc-url-index-s3`, `cc-url-index-hf`, `*.parquet`) + or AWS Athena (`athena://...`): `--hostname`/`--domain`/`--crawl`/`--limit` or your own `--sql`, cost confirmation (`--yes`), `--dry-run` + + `cdxt repackage` copies WARC records byte-for-byte with the async WARC fetcher (from the feat/warc-range-sources branch): + reads from https://, s3:// and hf:// (Hugging Face Storage Buckets), writes to local, s3:// or hf://, + `--processes`, `--parallel-readers`, `--add-metadata-record`, glob input; output files are now named `-001.warc.gz` + (`--wb` wayback replays keep the old writer and names); with `--processes`, the processes share one file counter + and write the final file names directly, so each `warcinfo` record names its own file + + `hf://` WARC reads resolve each WARC file once and read its records from the signed CDN URL + (one HF Resolvers rate-limit request per WARC file instead of per record) + + `cdxt index` reads CDX files (local or remote, globs): `--url`/`--hostname`/`--domain`/`--url-file` matched as SURT + prefixes (`--url-file-type domain` for host lists), one output per CDX file for an `-o` path without extension, + `--processes` + + SQL indexes: `--url` patterns (matched against `url_surtkey`), `--from`/`--to` (fetch time), `--language` (primary content language, three-letter ISO 639-3 codes) + + `--output-format cdx` / `--index-format cdx` as another name for `cdxj`; clear error for globs over http(s) + + Parquet output for `cdxt index` and Parquet input for `cdxt repackage` (`cdx_toolkit[parquet]`) + + `cdxt repackage`: default `--prefix REPACKAGE` (was `TEST`), no WARC file is written without captures, + a full URL in `warc_filename`/`filename` is used as it is (the `warc_url` column is gone) + + fixed `--to` with a day (e.g. `--to 20240410`), which was padded to the end of the month, and February in leap years + + removed the deprecated `cdx_iter` and `cdx_size` scripts + + new extras: `athena`, `duckdb`, `hf`, `parquet`; `fsspec` and `aiofiles` are now required + - 0.9.38 + deprecated support for py3.7 and py.3.8 + added support for py3.13 and py3.14 diff --git a/README.md b/README.md index bab194d..20a61e7 100644 --- a/README.md +++ b/README.md @@ -27,25 +27,114 @@ or clone this repo and use `pip install .` Optional extras: ``` -$ pip install cdx_toolkit[s3] # enable S3 and other remote filesystem support -$ pip install cdx_toolkit[all] # install all optional dependencies +$ pip install cdx_toolkit[s3] # S3 input/output and WARC downloads from s3:// +$ pip install cdx_toolkit[duckdb] # query the columnar URL index with DuckDB +$ pip install cdx_toolkit[athena] # query the columnar URL index with AWS Athena +$ pip install cdx_toolkit[hf] # Hugging Face Storage Buckets (hf://) +$ pip install cdx_toolkit[parquet] # Parquet output (and input of cdxt repackage) +$ pip install cdx_toolkit[all] # install all optional dependencies ``` ## Command-line tools +The `cdxt` command line tool has two commands: `index` selects captures +from an index, and `repackage` fetches the selected captures and writes +them into new WARC files. + ``` -$ cdxt --cc size 'commoncrawl.org/*' -$ cdxt --cc --limit 10 iter 'commoncrawl.org/*' # returns the most recent year -$ cdxt --crawl 3 --limit 10 iter 'commoncrawl.org/*' # returns the most recent 3 crawls -$ cdxt --cc --limit 10 --filter '=status:200' iter 'commoncrawl.org/*' +$ cdxt index cc --url 'commoncrawl.org/*' --limit 10 # returns the most recent year +$ cdxt index cc --url 'commoncrawl.org/*' --crawl 3 --limit 10 # returns the most recent 3 crawls +$ cdxt index cc --url 'commoncrawl.org/*' --limit 10 --filter '=status:200' +$ cdxt index cc --domain commoncrawl.org --limit 10 -o captures.csv +$ cdxt index cc --url 'commoncrawl.org/*' --count # estimated number of captures + +$ cdxt index ia --url 'commoncrawl.org/*' --limit 10 # will show the beginning of IA's crawl + +$ cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.csv # DuckDB +$ cdxt index cc-url-index-s3 --sql homepages.sql --yes -o captures.csv -$ cdxt --ia --limit 10 iter 'commoncrawl.org/*' # will show the beginning of IA's crawl -$ cdxt --ia --limit 10 warc 'commoncrawl.org/*' +$ cdxt index cc --url 'commoncrawl.org/*' --limit 10 -o captures.jsonl +$ cdxt repackage captures.jsonl --prefix COMMONCRAWL +$ cdxt index ia --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --wb ia --prefix IA ``` -cdxt takes a large number of command line switches, controlling -the time period and all other CDX query options. cdxt can generate -WARC, jsonl, and csv outputs. +### cdxt index + +`cdxt index INDEX` takes one of these indexes (the type is detected from INDEX, or set with +`--index-format cdx|duckdb|athena`): + +| INDEX | Index | +|---|---| +| `cc`, `ia` | Common Crawl or Internet Archive CDX API | +| `https://...` (no file extension) | another CDX API server, e.g. `https://web.archive.org/cdx/search/cdx` | +| `cc-url-index-s3` | Common Crawl's [columnar URL index](https://commoncrawl.org/blog/index-to-warc-files-and-urls-in-columnar-format) on S3 (`s3://commoncrawl/cc-index/table/cc-main/warc/`), queried with DuckDB; needs AWS credentials and `cdx_toolkit[duckdb]` | +| `cc-url-index-hf` | the same index on Hugging Face (`hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/`, recent crawls only), queried with DuckDB; needs `cdx_toolkit[duckdb,hf]`. Much slower than S3 | +| `*.parquet`, or a hive-partitioned directory with `--index-format duckdb` | another columnar index, local or remote, queried with DuckDB | +| `athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://bucket/path/` | the columnar URL index queried with AWS Athena (database `ccindex`, table `ccindex`; add `&table=...` for another table); needs `cdx_toolkit[athena]` | +| other paths, e.g. `'s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz'` | CDX files (one capture per line, optionally gzipped), local or remote, a single file or a glob | + +Captures are selected with `--url` (a URL pattern like `example.com/*` +or `*.example.com`, or an exact URL), `--hostname` (an exact hostname like +`www.example.com`) and `--domain` (a domain including its subdomains). +These can be repeated and combined; a capture is selected if it +matches any of them. All other options, like `--crawl`, `--from`, +`--to`, `--filter` and `--url-fgrep`, narrow the selection down. +Options an index does not support (e.g. `--filter` with DuckDB) are rejected. +For CDX files and the SQL indexes, `--url` patterns are matched against SURT +keys (`url_surtkey` in the columnar index), like the CDX API does: `www.` is +ignored, so `--url 'example.com/*'` also matches `www.example.com`, while +`--hostname example.com` does not. + +CDX files also support `--url-file PATH`: URL patterns (with the same syntax +as `--url`, e.g. `*.example.com` for a domain) from a file with one pattern per +line, or from the `url` column (or `--url-column NAME`) of a CSV, TSV, JSONL or +Parquet file. For lists of hosts, use `--url-file-type domain`: then an entry +like `example.com` matches the domain and its subdomains, and an entry with a +path like `example.com/reports` all URLs starting with it. The patterns are matched against the SURT keys of the CDX lines, +so long URL lists are fast. For DuckDB and Athena, join the index with a table +of the URLs in a custom `--sql` query instead. + +With a glob of CDX files and an `-o` path without file extension, `cdxt index` +writes one output per CDX file, with the same relative path and the selected +lines unchanged (e.g. to filter a whole crawl's index for a URL list), and +`--processes N` filters N files in parallel. No file is written for CDX files +without matches: + +``` +$ cdxt index 's3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz' \ + --url-file eot-hosts.txt --url-file-type domain -o s3://my-bucket/eot/CC-MAIN-2024-30/indexes --processes 8 +$ cdxt repackage 's3://my-bucket/eot/CC-MAIN-2024-30/indexes/cdx-*.gz' --prefix EOT-2024 +``` + +The SQL indexes (DuckDB, Athena) also support `--language CODE` (repeatable): +captures whose primary content language (the first of the `languages` field) +is one of the given [ISO 639-3](https://iso639-3.sil.org/) codes, e.g. `eng`, +`deu` or `fra` (three letters; e.g. `en` is rejected). + +The columnar index can also be queried with your own SQL: `--sql query.sql` +(or `--sql -` for stdin) runs the query as it is, instead of `--hostname`, +`--domain`, `--crawl`, `--from`, `--to`, `--language` and `--limit`. Without `--sql`, the query selects the +CDX fields (`url`, `timestamp`, `status`, `mime`, `digest`, `filename`, +`offset`, `length`, `languages`, ... and `crawl`) for the given hosts and +domains; `--dry-run` prints it without running it. + +SQL queries that are not restricted to at most 10 crawls (with `--crawl`, +or any `--sql` query) ask for confirmation first, because Athena bills per +TB scanned and DuckDB would read a lot of data. Pass `--yes` to skip the +question, e.g. in scripts. + +The captures are written to stdout or to the file given with +`-o/--output` (local or remote, e.g. `s3://...` with `cdx_toolkit[s3]`; +gzipped if the name ends with `.gz`). The output format is JSONL, CDXJ, +CSV, TSV or Parquet (needs `cdx_toolkit[parquet]`, not for stdout), detected +from the file extension or set with `--output-format`; stdout defaults to +JSONL. `cdx` is the same as `cdxj`: Common Crawl's index files have the `.cdx` +extension but use the CDXJ format (crawl index JSON). Use `--fields url,status,timestamp` to select fields; by default all +fields are written. + +`--count` prints the number of captures instead. For the CDX API this is +a cheap estimate based on the number of index pages; for SQL indexes it is +an exact `count(*)`. If you don't specify much about the crawls or dates or number of records you're interested in, some default limits will kick in to @@ -54,12 +143,79 @@ of 1000 records (`--limit 1000`) and a limit of 1 year of CC indexes. To exceed these limits, use `--limit` and `--crawl` or `--from` and `--to`. -If it seems like nothing is happening, add `-v` or `-vv` at the start: +If it seems like nothing is happening, add `-v` or `-vv`: ``` -$ cdxt -vv --cc size 'commoncrawl.org/*' +$ cdxt -vv index cc --url 'commoncrawl.org/*' --count ``` +### cdxt repackage + +`cdxt repackage INDEX_PATH` reads captures written by `cdxt index` (a +local or remote file, a glob like `s3://bucket/indexes/*.cdx.gz`, or stdin +if no path or `-` is given) and writes them into new WARC files. The input +format (JSONL, CDXJ, CSV or TSV) is detected from the first line, so Common +Crawl's CDX files can be used directly; `*.parquet` files are read as Parquet. +The columnar index column names (`warc_filename`, `warc_record_offset`, +`warc_record_length`) work too. A `filename` (or `warc_filename`) that is a +full URL (`s3://...`, `https://...`, `hf://...`) is used as it is, otherwise +it is relative to `--warc-download-prefix`. + +Captures with a WARC location (the `filename`, `offset` and `length` +fields, e.g. all Common Crawl captures) are copied byte-for-byte from +`--warc-download-prefix`: `https://data.commoncrawl.org` (default), +`s3://commoncrawl` (fastest in AWS us-east-1, needs `cdx_toolkit[s3]`) or +`hf://buckets/commoncrawl/commoncrawl` (needs `cdx_toolkit[hf]`). They are +written to `[-]-001.warc.gz`, `-002`, ..., where the +prefix can be a local path, `s3://...` or `hf://buckets/...` (default: +`REPACKAGE` in the current directory, e.g. `--prefix out/CC-HOMEPAGES` writes +`out/CC-HOMEPAGES-001.warc.gz`). No file is written when there are no +captures. The captures are +sorted by WARC file and offset first (for faster reads); `--no-sort` keeps +the input order and streams large inputs without keeping them in memory. + +Reads from `hf://` count against the Hugging Face Hub's +[Resolvers rate limit](https://huggingface.co/docs/hub/en/rate-limits) with one +request per WARC file (the records themselves are read from the CDN), so set +`HF_TOKEN` (anonymous access has the lowest limit). + +For large jobs, use `--parallel-readers N` (concurrent reads per process) +and `--processes N` (e.g. the number of vCPUs). With several processes, +the captures are split by WARC file across the processes, which write into +one shared `-001`, `-002`, ... series (which process writes which file +depends on timing; each file is a complete WARC). See [docs/notes/warc-fetcher-performance.md](docs/notes/warc-fetcher-performance.md) +for benchmarks and tuning. + +Captures without a WARC location, like Internet Archive captures, are +replayed from the wayback machine given with `--wb` (`ia` is a shortcut for +`https://web.archive.org/web`) and written one by one to +`[-]-000000.extracted.warc.gz`, ... + +Use `--size` (e.g. `100MB` or `1GiB`) to set the target size of each WARC +file, `--creator`, `--operator`, `--is-part-of` and `--description` to fill +in the `warcinfo` record, and `--add-metadata-record FILE` to add a file +(e.g. the URL list of the job) as a metadata record to each WARC file. + +### Migrating from the old commands + +`cdxt iter` is a deprecated alias of `cdxt index` with the same (new) +arguments; it prints a deprecation warning. The `size` and `warc` commands +have been removed; running them prints the equivalent new command. See the [migration guide](docs/cli-migration-guide.md) +for all changes and more examples. + +| Old | New | +|---|---| +| `cdxt --cc --limit 10 iter 'example.com/*'` | `cdxt index cc --url 'example.com/*' --limit 10` | +| `cdxt --crawl 3 iter 'example.com/*'` | `cdxt index cc --url 'example.com/*' --crawl 3` | +| `cdxt --source URL iter 'example.com/*'` | `cdxt index URL --url 'example.com/*'` | +| `cdxt --cc size 'example.com/*'` | `cdxt index cc --url 'example.com/*' --count` | +| `cdxt --ia warc 'example.com/*'` | `cdxt index ia --url 'example.com/*' \| cdxt repackage --wb ia` | + +All options now go after the command. `iter --csv`/`--jsonl` became +`--output-format`, all fields are written by default (`--all-fields` is +gone), `--get` is no longer needed (`--closest` uses a single query), and +`--url-fgrep`/`--url-fgrepv` moved from `warc` to `index`. + ## Selecting particular CCF crawls Common Crawl's data is divided into "crawls", which were yearly at the @@ -71,7 +227,7 @@ covers all of these crawls -- we have 1 index per crawl. The way you ask for a particular crawl is: ``` -$ cdxt --crawl CC-MAIN-2024-33 iter 'commoncrawl.org/*' +$ cdxt index cc --crawl CC-MAIN-2024-33 --url 'commoncrawl.org/*' ``` - `--crawl CC-MAIN-2024-33` is a single crawl. @@ -86,7 +242,7 @@ this index at [the blog post about it](https://commoncrawl.org/blog/index-to-war The Internet Archive cdx index is organized as a single crawl that goes from the very beginning until now. That's why there is no `--crawl` for -`--ia`. Note that cdx queries to `--ia` will default to one year year +`ia`. Note that cdx queries to `ia` will default to one year year and limit 1000 entries if you do not specify `--from`, `--to`, and `--limit`. ## Selecting by time @@ -96,7 +252,7 @@ Common Crawl queries, but for the Internet Archive you'll need to specify a time range like this: ``` -$ cdxt --ia --limit 1 --from 2008 --to 200906302359 iter 'commoncrawl.org/*' +$ cdxt index ia --limit 1 --from 2008 --to 200906302359 --url 'commoncrawl.org/*' ``` In this example the time range starts at the beginning of 2008 and @@ -108,15 +264,11 @@ most recent year. ``` $ cdxt --help -$ cdxt iter --help -$ cdxt warc --help -$ cdxt size --help +$ cdxt index --help +$ cdxt repackage --help ``` -for full details. Note that argument order really matters; each switch -is valid only either before or after the {iter,warc,size} command. - -Add -v (or -vv) to see what's going on under the hood. +for full details. Add -v (or -vv) to see what's going on under the hood. ## Python programming example @@ -155,7 +307,7 @@ to write into an extracted WARC file in [examples/iter-and-warc.py](examples/ite Filters can be used to limit captures to a subset of the results. -Any field name listed in `cdxt iter --all-fields` can be used in a +Any field name in the output of `cdxt index` can be used in a filter. These field names are appropriately renamed if the source is 'ia'. The different syntax of filter modifiers for 'ia' and 'cc' is not fully abstracted away by cdx_toolkit. @@ -174,7 +326,7 @@ Multiple filters will be combined with AND. For example, to limit captures to those which do not have status 200 and do not have status 404, ``` -$ cdxt --cc --filter '!=status:200' --filter '!=status:404' iter ... +$ cdxt index cc --filter '!=status:200' --filter '!=status:404' ... ``` Note that filters that discard large numbers of captures put a high diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md new file mode 100644 index 0000000..6a4c8d1 --- /dev/null +++ b/docs/cli-migration-guide.md @@ -0,0 +1,303 @@ +# Migrating to the new `cdxt` command line + +The `cdxt` command line tool has a new design with two commands: + +- **`cdxt index`** selects captures from an index and writes them as JSONL, CDXJ, CSV or TSV + (replaces `cdxt iter` and `cdxt size`). +- **`cdxt repackage`** fetches the captures listed in an index file and writes them into WARC files + (replaces `cdxt warc`, together with `cdxt index`). + +`cdxt iter` is now a deprecated alias of `cdxt index`: it takes exactly the same (new) arguments, +works the same and prints a deprecation warning. The old `iter` command lines (options before the +command, the URL as a positional argument) no longer work, rewrite them as shown below. + +``` +$ cdxt iter cc --url 'commoncrawl.org/*' --limit 10 +cdxt: "iter" is deprecated and will be removed in a future version, use "cdxt index" (same arguments). ... +{"digest": ..., "url": "https://commoncrawl.org/", ...} +``` + +The `size` and `warc` commands have been removed. If you run one of them, `cdxt` exits with an +error and prints the equivalent new command: + +``` +$ cdxt --cc --limit 10 warc 'commoncrawl.org/*' +cdxt: the "warc" command has been removed, use "cdxt index | cdxt repackage" instead. + +The equivalent command is: + + cdxt index cc --url 'commoncrawl.org/*' --limit 10 | cdxt repackage + +See "cdxt index --help", "cdxt repackage --help" or https://github.com/commoncrawl/cdx_toolkit#command-line-tools +``` + +## Quick reference + +| Old | New | +|---|---| +| `cdxt --cc iter URL` | `cdxt index cc --url URL` | +| `cdxt --ia iter URL` | `cdxt index ia --url URL` | +| `cdxt --source CDX_API_URL iter URL` | `cdxt index CDX_API_URL --url URL` | +| `cdxt --crawl 3 iter URL` | `cdxt index cc --crawl 3 --url URL` | +| `cdxt --cc size URL` | `cdxt index cc --url URL --count` | +| `cdxt --cc warc URL` | `cdxt index cc --url URL \| cdxt repackage` | +| `cdxt --ia warc URL` | `cdxt index ia --url URL \| cdxt repackage --wb ia` | + +## What changed + +### All options go after the command + +Before, the index options (`--cc`, `--limit`, `--from`, ...) had to come *before* the command and the +output options *after* it. Now all options go after the command, in any order. Only `-v`/`-vv` +and `--version` can also come before the command. + +```sh +# old +cdxt --cc --limit 10 --from 2017 iter --csv 'commoncrawl.org/*' + +# new +cdxt index cc --url 'commoncrawl.org/*' --limit 10 --from 2017 --output-format csv +``` + +### The index is a positional argument + +`--cc`, `--ia` and `--source URL` are replaced by the first argument of `cdxt index`: + +| Old | New | +|---|---| +| `--cc` | `cdxt index cc` | +| `--ia` | `cdxt index ia` | +| `--source https://web.archive.org/cdx/search/cdx` | `cdxt index https://web.archive.org/cdx/search/cdx` | +| `--crawl ...` (implied `--cc`) | `cdxt index cc --crawl ...` | + +`--crawl` now has to be combined with `cc` explicitly; it cannot be used with other indexes. + +### The URL is an option + +The URL pattern moves from a positional argument to `--url`. `--url` can be repeated, and there +are two new ways to select captures: + +| Option | Selects | Example | +|---|---|---| +| `--url PATTERN` | a URL pattern (as before) | `--url 'commoncrawl.org/*'`, `--url '*.commoncrawl.org'` | +| `--hostname HOST` | all captures of one exact hostname | `--hostname www.commoncrawl.org` | +| `--domain DOMAIN` | all captures of a domain and its subdomains | `--domain commoncrawl.org` | + +They can be repeated and combined; a capture is selected if it matches any of them: + +```sh +cdxt index cc --domain commoncrawl.org --url 'example.com/about*' --limit 100 +``` + +### Output: file or stdout, format and fields + +| Old | New | +|---|---| +| output always to stdout | stdout by default, or `-o/--output PATH` (local or remote, e.g. `s3://...`; gzipped if it ends with `.gz`) | +| `iter` (text: `status 200, timestamp ..., url ...`) | JSONL by default | +| `iter --jsonl` | default, or `--output-format jsonl` | +| `iter --csv` | `--output-format csv`, or `-o captures.csv` | +| — | `--output-format cdxj` / `tsv`, or `-o captures.cdxj` / `captures.tsv` | +| `--fields url,status,timestamp` (default) | `--fields url,status,timestamp` (default: all fields) | +| `--all-fields` | default | + +The output format is detected from the `--output` file extension (`.jsonl`, `.json`, `.cdxj`, +`.cdx`, `.csv`, `.tsv`, each optionally with `.gz`). For other extensions, set `--output-format`. + +The old human-readable text output is gone. The closest replacement is CSV with the old default fields: + +```sh +# old +cdxt --cc --limit 10 iter 'commoncrawl.org/*' + +# new +cdxt index cc --url 'commoncrawl.org/*' --limit 10 --fields url,status,timestamp --output-format csv +``` + +Note that the CSV columns now follow the order given in `--fields` (before, they were sorted +alphabetically) and lines end with `\n` instead of `\r\n`. + +### `size` is now `index --count` + +`--count` prints the same estimated number of captures as the old `size` command (based on the number +of index pages, so it is cheap but not exact). + +```sh +# old +cdxt --cc size 'commoncrawl.org/*' + +# new +cdxt index cc --url 'commoncrawl.org/*' --count +``` + +`size --details` (one line per index) has been removed. Use `-v` to see which indexes are queried. + +### `warc` is now `index` + `repackage` + +Creating WARC files is now two steps: `cdxt index` selects the captures, and `cdxt repackage` +fetches them and writes the WARC files. You can pipe one into the other, or save the captures to a +file first (handy to inspect them, or to repackage the same selection again later). + +```sh +# old +cdxt --cc --limit 10 warc --prefix COMMONCRAWL 'commoncrawl.org/*' + +# new: piped +cdxt index cc --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --prefix COMMONCRAWL + +# new: via a file +cdxt index cc --url 'commoncrawl.org/*' --limit 10 -o captures.jsonl +cdxt repackage captures.jsonl --prefix COMMONCRAWL +``` + +The `warc` options are split between the two commands: + +| Old `warc` option | New | +|---|---| +| `--url-fgrep TEXT` | `cdxt index --url-fgrep TEXT` | +| `--url-fgrepv TEXT` | `cdxt index --url-fgrepv TEXT` | +| `--prefix` (default `TEST`), `--subprefix` | `cdxt repackage --prefix` (default `REPACKAGE`), `--subprefix` | +| `--size 1000000000` | `cdxt repackage --size 1000000000`, or human readable: `--size 1GB`, `--size 100MiB` | +| `--creator`, `--operator` | `cdxt repackage --creator`, `--operator` | +| `--warc-download-prefix` | `cdxt repackage --warc-download-prefix` | +| `--wb` (global option) | `cdxt repackage --wb` | +| — | `cdxt repackage --is-part-of`, `--description` (`warcinfo` record fields) | + +**Common Crawl** captures contain their WARC location (`filename`, `offset`, `length`) and are +fetched from `--warc-download-prefix` (default `https://data.commoncrawl.org`; use +`s3://commoncrawl` in AWS us-east-1). Two things differ from the old `warc` command: + +- **File names:** `[-]-001.warc.gz`, `-002`, ... instead of + `[-]-000000.extracted.warc.gz`, `-000001`, ... +- **No empty WARC file** is written when the index selected no captures. +- **Records are copied byte-for-byte** from the source WARC files. Before, they were parsed and + re-written with added `WARC-Source-URI` and `WARC-Source-Range` headers; now the records (and + their digests) are exactly the originals. + +The records are also sorted by WARC file and offset before fetching (`--no-sort` keeps the order of +the index), and can be fetched in parallel (`--parallel-readers`, `--processes`). + +**Internet Archive** captures have no WARC location; they are replayed from the wayback machine. +Before, `--ia` did this automatically. Now pass `--wb ia` (or the URL of another wayback machine). +These are written as before, to `[-]-000000.extracted.warc.gz`, ...: + +```sh +# old +cdxt --ia --limit 10 warc 'commoncrawl.org/*' + +# new +cdxt index ia --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --wb ia + +# old: other CDX server and wayback machine +cdxt --source https://web.archive.org/cdx/search/cdx --wb https://web.archive.org/web warc 'commoncrawl.org/*' + +# new +cdxt index https://web.archive.org/cdx/search/cdx --url 'commoncrawl.org/*' \ + | cdxt repackage --wb https://web.archive.org/web +``` + +### Other option changes + +| Old | New | +|---|---| +| `--get` | removed; `cdxt index` always picks the right query type (`--closest` uses a single query) | +| `--closest TS --get` | `--closest TS` | +| `--from`, `--to`, `--filter`, `--limit` | unchanged (after the command) | +| `--cc-mirror`, `--cc-sort` | unchanged (after the command); `--cc-sort` now actually takes effect | +| `-v`, `-vv` | unchanged; before or after the command | + +`--limit` is the maximum number of captures read from the index. As with the old `warc --url-fgrep`, +the limit applies *before* `--url-fgrep`/`--url-fgrepv`, so the output can have fewer records. + +Usage errors (for example a missing `--url`, or `--crawl` with `ia`) now print a short message and +exit with code 2 instead of showing a Python traceback. + +## More examples + +```sh +# most recent 3 crawls +# old: cdxt --crawl 3 --limit 10 iter 'commoncrawl.org/*' +cdxt index cc --crawl 3 --url 'commoncrawl.org/*' --limit 10 + +# a specific crawl +# old: cdxt --crawl CC-MAIN-2024-33 iter 'commoncrawl.org/*' +cdxt index cc --crawl CC-MAIN-2024-33 --url 'commoncrawl.org/*' + +# time range at the Internet Archive +# old: cdxt --ia --limit 1 --from 2008 --to 200906302359 iter 'commoncrawl.org/*' +cdxt index ia --url 'commoncrawl.org/*' --limit 1 --from 2008 --to 200906302359 + +# filters +# old: cdxt --cc --filter '!=status:200' --filter '!=status:404' iter 'commoncrawl.org/*' +cdxt index cc --url 'commoncrawl.org/*' --filter '!=status:200' --filter '!=status:404' + +# selected fields as CSV +# old: cdxt --cc --limit 10 iter --fields=digest,length,offset --csv 'commoncrawl.org/*' +cdxt index cc --url 'commoncrawl.org/*' --limit 10 --fields digest,length,offset -o captures.csv + +# closest capture +# old: cdxt --cc --limit 3 --get --closest=20170615 iter 'commoncrawl.org/*' +cdxt index cc --url 'commoncrawl.org/*' --limit 3 --closest 20170615 + +# WARC without robots.txt, with metadata, in 100 MB files +# old: cdxt --cc --limit 100 warc --prefix CC --subprefix BLOG --size 100000000 \ +# --creator me --url-fgrepv robots.txt 'commoncrawl.org/*' +cdxt index cc --url 'commoncrawl.org/*' --limit 100 --url-fgrepv robots.txt \ + | cdxt repackage --prefix CC --subprefix BLOG --size 100MB --creator me + +# save the selection to S3, repackage it later (needs cdx_toolkit[s3]) +cdxt index cc --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o s3://my-bucket/captures.jsonl.gz +cdxt repackage s3://my-bucket/captures.jsonl.gz --prefix s3://my-bucket/warcs/COMMONCRAWL +``` + +## New: the columnar URL index + +Besides the CDX API, `cdxt index` can query Common Crawl's columnar URL index (parquet) with DuckDB +or AWS Athena. This was not possible with the old commands: + +```sh +# DuckDB over s3://commoncrawl/cc-index/table/cc-main/warc/ (needs AWS credentials and cdx_toolkit[duckdb]) +cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.jsonl + +# your own SQL query +cdxt index cc-url-index-s3 --sql homepages.sql --yes -o homepages.csv + +# AWS Athena (needs cdx_toolkit[athena]) +cdxt index 'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://my-bucket/athena/' \ + --hostname commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.jsonl +``` + +See the README for all index types. + +## Users of the `feat/warc-range-sources` branch + +The `repackage` command of the unreleased `feat/warc-range-sources` branch (#73) is split into +`cdxt index` (selecting the records) and `cdxt repackage` (fetching them): + +| Branch | New | +|---|---| +| `cdxt --crawl C repackage --target-source sql --engine athena --athena-database D --athena-s3-output S3 --hostnames H ...` | `cdxt index 'athena://athena.us-east-1.amazonaws.com/D?S3OutputLocation=S3' --crawl C --hostname H -o captures.csv` + `cdxt repackage captures.csv ...` | +| `... --engine duckdb --duckdb-index-path PATH` | `cdxt index PATH --index-format duckdb ...` (or `cc-url-index-s3`, `cc-url-index-hf`) | +| `--hostnames H1 H2`, `--domains D1 D2` | `--hostname H1 --hostname H2`, `--domain D1 --domain D2` | +| `--query SQL`, `--query-file FILE` | `cdxt index ... --sql FILE` (`--sql -` reads stdin) | +| — | new for SQL indexes: `--from`/`--to` (`fetch_time`), `--language` (primary content language, ISO 639-3), Parquet output (`-o captures.parquet`, `cdx_toolkit[parquet]`) | +| `--confirm-cost` | `cdxt index --yes` | +| `--target-source cdx --cdx-path PATH [--cdx-glob GLOB]` | `cdxt repackage 'PATH/GLOB'` (CDX files are read directly) | +| `--target-source csv --csv-path FILE` | `cdxt repackage FILE` | +| `--range-jobs-output FILE --no-fetch` | `cdxt index ... -o FILE` | +| `--csv-self-contained` (`warc_url` column) | removed; `warc_filename` (or `filename`) may hold a full URL (`s3://`, `https://`, `hf://`), which is used as it is, otherwise it is joined with `--warc-download-prefix` | +| `--limit N` (global option) | `cdxt index --limit N` (repackage does not filter) | +| `--parallel N`, `--parallel_readers N` | `cdxt repackage --parallel-readers N` | +| `--no-sort-ranges` | `cdxt repackage --no-sort` | +| `--write-paths-as-metadata-records F1 F2` | `cdxt repackage --add-metadata-record F1 --add-metadata-record F2` | +| `--log_every_n N` | `cdxt repackage --log-every-n N` | +| `--keep-shards` | removed; with `--processes`, all processes write directly into one `-001`, `-002`, ... series (`--subprefix` now works too) | +| `--hf-reader fsspec\|cdn` | removed; `hf://` WARCs are always read from the CDN, with one rate-limited `/resolve/` request per WARC file instead of per record (`fsspec` was ~10x slower and dropped records under load in the benchmarks) | +| `--processes`, `--warc-download-prefix`, `--prefix`, `--subprefix`, `--size`, `--creator`, `--operator`, `--is-part-of`, `--description` | unchanged (`--size` also accepts e.g. `1GB`) | +| `cdxt filter_cdx INPUT_BASE FILTER_FILE OUTPUT_BASE --input-glob GLOB --parallel N` | `cdxt index 'INPUT_BASE/GLOB' --url-file FILTER_FILE -o OUTPUT_BASE --processes N` (one output per CDX file). Add `--url-file-type domain` to match the entries like `--filter-type url` did (`example.com` = the domain with subdomains, entries with a path = URL prefixes); without it, the entries are URL patterns like `--url` | +| `--filter-type surt`, `--overwrite`, `--limit` (per file) | removed; outputs are overwritten, `--limit` only works with a single output | + +## Python API + +The Python API (`cdx_toolkit.CDXFetcher`, `cdx.iter()`, `cdx_toolkit.warc.get_writer()`, ...) has +not changed. Only the command line tool is affected. diff --git a/docs/notes/warc-fetcher-performance.md b/docs/notes/warc-fetcher-performance.md new file mode 100644 index 0000000..2bd1c01 --- /dev/null +++ b/docs/notes/warc-fetcher-performance.md @@ -0,0 +1,201 @@ +# WARC fetcher performance: bottlenecks, analysis & recommendations + +Notes on optimizing the `cdxt repackage` fetch pipeline for large jobs (e.g. a +range-jobs file with ~1M rows), with a focus on running on EC2 in-region against +`s3://commoncrawl`. + +Status legend: **[done]** implemented on this branch · **[proposed]** not yet implemented. + +> These notes were written on the `feat/warc-range-sources` branch (#73). In the new CLI the +> range-job sources became `cdxt index` (CDX API, Athena, DuckDB) and some options were renamed: +> `--parallel`/`--parallel_readers` → `--parallel-readers`, `--no-sort-ranges` → `--no-sort`, +> `--write-paths-as-metadata-records` → `--add-metadata-record`, `--confirm-cost` → `cdxt index --yes`. +> `--hf-reader` was removed: `hf://` is always read with the CDN reader, which now resolves each +> WARC file once instead of once per record. + +--- + +## How the fetcher works today + +Three async stages on a single `asyncio` event loop (`cdx_toolkit/repackage/warc_filter.py`): + +1. **Produce** — the selected `RangeJobSource` yields `RangeJob`s (one per WARC record: + url + byte offset + length). The source is a *sync* generator run in a worker + thread (`asyncio.to_thread(drain)`, `warc_filter.py:~276`), pushing each job onto a + bounded `range_jobs_queue` via `run_coroutine_threadsafe(...).result()`. +2. **Read** — `num_readers` reader coroutines (`= --parallel`) each pull a job and do + **one S3 `GetObject` (or HTTP GET) per job** (`read_warc_records` → + `RangeJob.ranged_get_bytes`, `warc_filter.py:~446`, `data_classes.py:64`). The + payload (raw, already-gzipped WARC record bytes) is enqueued. +3. **Write** — `num_writers` (`= num_readers/6`) writer coroutines each own one output + shard and append records to it (`write_warc_records`, `warc_filter.py:~497`). + +Clients/config: + +- One shared `aioboto3` S3 read client and one write client (`get_aws_clients`, + `warc_filter.py:~179`); read pool sized `num_readers*3`. +- Retries/backoff for throttling (503 SlowDown) in `with_retries` (`s3_utils.py:27`). +- Output writers: `S3ShardWriter` (multipart upload, ≥5 MiB parts) or `LocalFileWriter` + (`aiofiles`); shard files keyed by `writer_id` + `sequence` + (`generate_warc_filename`, `warc_utils.py:67`). Records are written **verbatim** — no + recompression. + +--- + +## Key insight: concurrency vs. cores + +`aioboto3`/`asyncio` give **I/O concurrency**, not **multi-core parallelism**: + +- With `--parallel=30` there are up to 30 S3 requests *in flight at once*; while most + are blocked on the network, others run. For network-bound work this is exactly right. +- But asyncio is single-threaded: **one event loop, one OS thread, one CPU core.** All + the *CPU* work — TLS encrypt/decrypt, HTTP framing, aiobotocore object churn, byte + copies — is serialized on that one core. The readers **and** writers are all + coroutines on the same loop (`asyncio.create_task(...)`, `warc_filter.py:~321` and + `~334`). +- The GIL means *threads* wouldn't help CPU-bound work either. Only **multiple + processes** use multiple cores. + +Corollary: **sharded output ≠ multi-core.** Having N writers means N concurrent MPU +streams on one core, not N cores. Adding writers (or readers) past the point where the +single core saturates buys nothing but scheduling overhead and memory. + +--- + +## What actually limits a 1M-row job in-region + +**Bandwidth is rarely the constraint.** 1M records at ~30 KB compressed ≈ ~30 GB; over +even 5 Gbps that is ~50 s, at 25 Gbps ~10 s. A c5n NIC is nowhere near the bottleneck. + +**Request rate on a single core usually is.** 1M independent range GETs, each with +per-request TLS + HTTP + Python overhead, all on one event-loop core. A single loop +typically sustains on the order of a few thousand small GET/s before that core +saturates → ~3–8 min CPU-bound on one core, with the NIC mostly idle and the box's other +vCPUs unused. + +Two regimes, and the optimizations differ: + +- **Many small requests** (default): single-core CPU/TLS bound → cut request count + (coalescing), lift the single-core ceiling (uvloop), or use more cores + (multi-process). +- **Fewer, larger reads** (after coalescing, or large records): network-bound → one core + drives multiple Gbps fine; more processes buy little. + +### How to tell which regime you're in + +Run a sample and watch `htop`/`mpstat` during the fetch: + +- One core pinned ~100 %, NIC well below capacity → **single-core CPU-bound** → + coalescing / uvloop / multi-process. +- All cores low, throughput still short → **not CPU-bound**: the limiter is concurrency, + request latency, or S3 throttling — process/writer count is irrelevant. + +--- + +## Writer review (why more output shards won't fix a read-bound core) + +- **No recompression** — records arrive already gzipped and are written verbatim + (`writer.write(item.data)`, `warc_filter.py:~577`). The writer adds no gzip CPU. +- **`S3ShardWriter`** (`s3_writer.py:84`) buffers into a `bytearray`; on each ≥5 MiB + boundary it does `buffer[:n]` (copy), `del buffer[:n]` (an O(remaining) memmove), + `bytes(chunk)` (another copy), then `await upload_part`. Real but modest byte-copy cost. +- **`LocalFileWriter`** (`local_writer.py`) is `aiofiles` with an 8 KiB buffer and a + `flush()` per write — on a fast read stream the per-flush overhead + EBS limits are the + more likely bottleneck (prefer writing to `s3://` on a c5n). +- **[done] One writer per process.** Because writing is a cheap verbatim copy that one + coroutine sustains, the old `num_writers = num_readers/6` multi-writer fan-out (multiple + shards on one event loop) bought nothing but scheduling overhead and was removed. Each + process now has exactly one writer = one output shard; multi-core scaling and sharding + come from `--processes` (multiple event loops), and shards are merged into one file. The + CPU hot spot is the **read side** (many small GETs + TLS), not the write side. + +The flip side: the shard model makes **multi-process essentially free on the output +side** — shards are independent files with no cross-shard state and no merge step, so N +worker processes can each write their own shards into the same prefix. + +--- + +## Recommendations (ranked) + +Assume parallelism is already configured (`--parallel` high) and reads go to +`s3://commoncrawl` in-region. + +1. **[done] Sort by `(warc_filename, warc_record_offset)`.** Groups records of the same + WARC file with ascending offsets → read locality, connection reuse, and the + prerequisite for coalescing and clean per-file process sharding. Implemented as an + `ORDER BY` on guided SQL queries (Athena/DuckDB) and an in-memory sort when loading a + CSV source; `--no-sort-ranges` opts out. Raw `--query` and `cdx` sources keep their + order. + +2. **[proposed] Coalesce adjacent ranges (highest remaining value).** After sorting, + merge nearby same-file ranges into one GET (fetch the superrange, slice locally by + offset). Safe because each WARC record is an independent gzip member, so concatenated + members remain valid and byte-exact slicing reconstructs each record. Use a gap + threshold (e.g. ≤256 KB–1 MB) to swallow the small request/metadata records between + indexed responses. Effect: collapses ~1M tiny latency/CPU-bound GETs into ~10–100k + larger sequential reads — directly attacks the request-rate ceiling; bandwidth easily + absorbs the gap bytes. Extreme case: a **density heuristic** — when a large fraction of + a ~1 GB file is needed, GET the whole file once and slice. + +3. **[done] Use all cores via multi-process sharding** — confirmed single-core CPU-bound + (a single loop saturates one core at ~450 small GET/s; one process pins ~1 core at + `--parallel 96`). Implemented natively as `cdxt repackage --processes N` + (`cdx_toolkit/repackage/multiprocess.py`): the source is drained once and sharded by + `warc_filename`, one worker process per core fetches its shard + (`--parallel_readers R`, one writer each → one shard/proc), and the shards are merged + into a single `.warc.gz` (`cdx_toolkit/repackage/merge.py`: server-side S3 + `UploadPartCopy`, or fsspec streaming locally). Measured on c5n.xlarge (4 vCPU): + ~457 → ~1130 rec/s fetch (2.6×, cores ~97 %) for a ~1 GiB / 35.7 k-record homepages job; + ~60 s end-to-end including source sharding and the merge. Sublinear because each + process's producer thread (CSV sort + per-row cross-thread enqueue) competes with its + event loop for the GIL — see latent issues. + +4. **[done] uvloop.** Gated by `CDXT_UVLOOP=1` in `WARCFilter.filter` (`warc_filter.py`). + Measured ~+8 % single-core; stacks under multi-process. + +5. **[proposed] Bound coalesced read size / watch RAM.** A c5n.xlarge has 10.5 GiB; + whole-file (~1 GB) reads × many readers will OOM. Cap superrange size and gap + threshold; revisit `warc_records_queue_size` (`warc_filter.py:~62`, default 200) since + each buffered payload is larger after coalescing. + +6. **[done] Write output to `s3://` in-region, not local EBS.** Measured: S3-direct MPU + output is within noise of local EBS at this write rate (~35 MB/s total) and skips the + EBS round-trip + separate upload, so prefer `--prefix s3://…`. On the writer count: at + in-region read rates a *single* writer per process keeps up easily (verbatim copy, + 5 MiB parts), so `--parallel_writers 1` is both fastest-enough and gives one shard per + process — better than the `writers = readers/6` default for this workload. + +7. **[proposed] Instance selection follows the constraint.** Because the binding limit is + request-rate/CPU (not bandwidth), **more/faster cores beat more Gbps**. c5n.xlarge's + "up to 25 Gbps" is burstable and depletes to baseline on long jobs anyway — but that + barely matters when 30 GB is seconds of transfer. Scale vCPUs to feed (3); reserve + sustained-25 Gbps types for genuinely bandwidth-bound (huge-record) jobs. + +8. **[proposed] Dedup identical `(filename, offset, length)` rows** before fetching — + free; fold into the sort/coalesce pass. + +### Latent issues worth noting + +- **HTTP path blocks the event loop.** `ranged_get_bytes`'s HTTP branch calls the + *synchronous* `myrequests_get(...)` without `await`/`to_thread` (`data_classes.py:~91`). + Since the default `--warc-download-prefix` is `https://data.commoncrawl.org`, the + default fetch path serializes the "parallel" readers. On EC2, prefer `s3://commoncrawl` + (the S3 branch is properly async); otherwise the HTTP fetch should be offloaded to a + thread pool / async HTTP client. **[proposed]** +- **Per-row cross-thread enqueue.** The producer does one + `run_coroutine_threadsafe(...).result()` per job (`warc_filter.py:~268`) — 1M + loop↔thread handoffs. Batching enqueues removes a real per-row overhead at this scale. + **[proposed]** + +--- + +## TL;DR + +1. **[done]** Sort by `(warc_filename, offset)` — locality + enables coalescing. +2. **[done]** Multi-process by filename (`--processes N`) — ~2.6× on 4 cores; shards merged + into one file. The big confirmed win in-region. +3. **[done]** uvloop (`CDXT_UVLOOP=1`), one writer per process, `s3://` output — supporting wins. +4. **[proposed]** Coalesce adjacent ranges, bounded read size, dedup — remaining ideas. + +Always **measure the regime first** (`htop` during a sample run): in-region the limiter +is almost always request-rate on one core, not bandwidth. From 316e96eb88470cd16064b40fcec8337f148aceed Mon Sep 17 00:00:00 2001 From: malteos Date: Fri, 9 Oct 2026 12:43:58 +0200 Subject: [PATCH 03/91] fix: CI on Windows and in the minimal job, test the untested paths CI - .gitattributes: no line-ending conversion for tests/data, so fixtures (e.g. the metadata record with a fixed payload digest) are byte-identical on Windows - compare relative paths with as_posix() in the per-file output test (Windows) - `make test` no longer runs examples/iter-and-warc.py against the live Common Crawl index (connections from CI runners time out; it also took ~45 minutes in the minimal job), it is now `make example` Fixes found by the new tests - WARC files: only rotate a file that already has records; a record larger than the remaining --size no longer leaves a WARC file with only the warcinfo record - DuckDB: retry S3 requests longer (8 retries, 1s, 2s, 4s, ...) on 503 SlowDown - is_s3_url() accepts None (WARCFilter without warc_download_prefix) Tests (coverage of the new code) - Athena source and `cdxt index athena://...` with a fake boto3 client - HF reader: resolve without redirect, resolve/read errors, 429s, cache eviction - DuckDB source with a local parquet file and a fake connection (HF filesystem) - WARC pipeline and multi-process mode in-process (fake HTTP/HF reads, memory:// output), reader/writer errors, uvloop - usage errors and edge cases of index/repackage Cleanup: remove dead code (fsspec fallbacks in record_io, the old --jsonl/--csv translation, the cost estimate of the repackage sources, Athena get_databases, the Python < 3.9 check, DuckDbSource's query builder) --- .gitattributes | 2 + Makefile | 5 +- cdx_toolkit/cli/deprecated.py | 4 - cdx_toolkit/indexes/base.py | 8 +- cdx_toolkit/indexes/sql.py | 6 +- cdx_toolkit/record_io.py | 26 +--- cdx_toolkit/repackage/s3_utils.py | 6 +- cdx_toolkit/repackage/sources/athena.py | 13 +- cdx_toolkit/repackage/sources/base.py | 9 +- cdx_toolkit/repackage/sources/duckdb.py | 60 ++------ cdx_toolkit/repackage/warc_filter.py | 33 ++--- tests/repackage/test_athena_source.py | 137 +++++++++++++++++ tests/repackage/test_duckdb_source.py | 68 +++++++++ tests/repackage/test_hf_utils.py | 87 +++++++++++ tests/repackage/test_warc_filter_pipeline.py | 146 +++++++++++++++++++ tests/unit/test_cdx_files.py | 2 +- tests/unit/test_usage_errors.py | 134 +++++++++++++++++ 17 files changed, 629 insertions(+), 117 deletions(-) create mode 100644 .gitattributes create mode 100644 tests/repackage/test_athena_source.py create mode 100644 tests/repackage/test_duckdb_source.py create mode 100644 tests/repackage/test_warc_filter_pipeline.py create mode 100644 tests/unit/test_usage_errors.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7b9efa3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# test fixtures are compared byte for byte (e.g. WARC payload digests): no line-ending conversion +tests/data/** -text diff --git a/Makefile b/Makefile index 8a60525..b33e273 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: init test clean_coverage test_coverage lint distclean distcheck dist install +.PHONY: init test example clean_coverage test_coverage lint distclean distcheck dist install init36: # packages are deprecating support, so this uses exact versions @@ -12,6 +12,9 @@ unit: test: PYTHONPATH=. py.test --doctest-modules cdx_toolkit tests -v -v + +# runs against the live Common Crawl index (not part of `test`: connections from CI runners time out) +example: PYTHONPATH=. examples/iter-and-warc.py clean_coverage: diff --git a/cdx_toolkit/cli/deprecated.py b/cdx_toolkit/cli/deprecated.py index 3633e5d..fed3d28 100644 --- a/cdx_toolkit/cli/deprecated.py +++ b/cdx_toolkit/cli/deprecated.py @@ -61,10 +61,6 @@ def translate(args: List[str]) -> str: index = value elif option in ('-v', '-vv', '--verbose'): index_args.append(option) - elif option == '--jsonl': - index_args += ['--output-format', 'jsonl'] - elif option == '--csv': - index_args += ['--output-format', 'csv'] elif option in INDEX_OPTIONS: index_args += [option, value] elif option in REPACKAGE_OPTIONS: diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py index 4d65808..85a05f8 100644 --- a/cdx_toolkit/indexes/base.py +++ b/cdx_toolkit/indexes/base.py @@ -29,15 +29,17 @@ def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: """Yield the selected captures as mappings with CDX field names (url, timestamp, filename, ...). Backends apply --limit themselves, so it can be pushed down to the index.""" - raise NotImplementedError + raise NotImplementedError # pragma: no cover + @abstractmethod def count(self, query: IndexQuery) -> int: """Cheap (possibly estimated) number of captures matching the query.""" - raise UsageError(f'--count is not supported for the {self.name} index') + raise NotImplementedError # pragma: no cover + @abstractmethod def explain(self, query: IndexQuery) -> str: """Describe what would be queried (--dry-run), without querying the index.""" - raise UsageError(f'--dry-run is not supported for the {self.name} index') + raise NotImplementedError # pragma: no cover def iter_index_records(backend: IndexBackend, query: IndexQuery) -> Iterator[Mapping]: diff --git a/cdx_toolkit/indexes/sql.py b/cdx_toolkit/indexes/sql.py index 87ce048..b066ccb 100644 --- a/cdx_toolkit/indexes/sql.py +++ b/cdx_toolkit/indexes/sql.py @@ -181,10 +181,10 @@ def validate(self, query: IndexQuery) -> None: def from_clause(self, crawls: Optional[List[str]]) -> Tuple[str, bool]: """FROM expression for the guided query, and whether it already prunes the crawls.""" - raise NotImplementedError + raise NotImplementedError # pragma: no cover def iter_sql_rows(self, sql: str) -> Iterator[dict]: - raise NotImplementedError + raise NotImplementedError # pragma: no cover def prepare(self, query: IndexQuery, order_by: bool = True) -> Tuple[str, Optional[int]]: """The SQL to run and the number of crawls it is restricted to (None = unknown).""" @@ -254,7 +254,7 @@ def from_clause(self, crawls): def iter_sql_rows(self, sql): from cdx_toolkit.repackage.sources.duckdb import DuckDbSource - source = DuckDbSource(query=sql, index_path=self.location, warc_download_prefix=None) + source = DuckDbSource(query=sql) return source.iter_rows() diff --git a/cdx_toolkit/record_io.py b/cdx_toolkit/record_io.py index 12ca453..5767947 100644 --- a/cdx_toolkit/record_io.py +++ b/cdx_toolkit/record_io.py @@ -1,22 +1,18 @@ ''' Reading and writing index records (captures) as JSONL, CDXJ, CSV, TSV or Parquet. -Paths can be local or remote (via fsspec, if installed), optionally gzipped; "-" is stdin/stdout. +Paths can be local or remote (via fsspec), optionally gzipped; "-" is stdin/stdout. Parquet needs the optional pyarrow dependency (cdx_toolkit[parquet]) and a file path. ''' import contextlib import csv -import gzip import itertools import json import re import sys from typing import IO, Iterable, Iterator, List, Mapping, Optional -try: - import fsspec -except ImportError: # pragma: no cover - exercised in minimal installs - fsspec = None +import fsspec TEXT_FORMATS = ('jsonl', 'cdxj', 'csv', 'tsv') @@ -72,13 +68,7 @@ def open_text(path: str, mode: str) -> IO: """Open a local or remote text file; '-' means stdin/stdout (which is not closed).""" if path == '-': return contextlib.nullcontext(sys.stdout if 'w' in mode else sys.stdin) - if fsspec is not None: - return fsspec.open(path, mode + 't', compression='infer', newline='') - if '://' in path: - raise RuntimeError(f'Reading or writing {path} requires fsspec. Install cdx_toolkit[s3].') - if path.endswith('.gz'): - return gzip.open(path, mode + 't', newline='') - return open(path, mode, newline='') + return fsspec.open(path, mode + 't', compression='infer', newline='') class RecordWriter: @@ -138,12 +128,6 @@ def expand_paths(path: str) -> List[str]: if not is_glob(path): return [path] check_glob(path) - if fsspec is None: - import glob - - if '://' in path: - raise RuntimeError(f'Reading {path} requires fsspec. Install cdx_toolkit[s3].') - return sorted(glob.glob(path)) fs, fs_path = fsspec.url_to_fs(path) return [fs.unstrip_protocol(p) for p in sorted(fs.glob(fs_path))] @@ -232,7 +216,7 @@ def write_parquet(records: Iterable[Mapping], path: str, fields: Optional[List[s n = 0 schema = None writer = None - with fsspec.open(path, 'wb') if fsspec is not None else open(path, 'wb') as fh: + with fsspec.open(path, 'wb') as fh: try: batch = [] for record in itertools.chain(records, [None]): @@ -261,7 +245,7 @@ def write_parquet(records: Iterable[Mapping], path: str, fields: Optional[List[s def read_parquet(path: str) -> Iterator[dict]: pa = import_pyarrow() - with fsspec.open(path, 'rb') if fsspec is not None else open(path, 'rb') as fh: + with fsspec.open(path, 'rb') as fh: for batch in pa.parquet.ParquetFile(fh).iter_batches(): for record in batch.to_pylist(): yield normalize_field_names({k: v for k, v in record.items() if v is not None}) diff --git a/cdx_toolkit/repackage/s3_utils.py b/cdx_toolkit/repackage/s3_utils.py index ddd48f3..5eb9470 100644 --- a/cdx_toolkit/repackage/s3_utils.py +++ b/cdx_toolkit/repackage/s3_utils.py @@ -1,6 +1,6 @@ import asyncio import logging -from typing import Tuple +from typing import Optional, Tuple from os import urandom try: @@ -27,8 +27,8 @@ def parse_s3_uri(uri: str) -> Tuple[str, str]: return rest[:i], rest[i+1:] -def is_s3_url(url: str) -> bool: - return url.startswith('s3:/') +def is_s3_url(url: Optional[str]) -> bool: + return bool(url) and url.startswith('s3:/') async def with_retries(coro_factory, *, op_name: str, max_attempts: int, base_backoff_seconds: float): diff --git a/cdx_toolkit/repackage/sources/athena.py b/cdx_toolkit/repackage/sources/athena.py index 2dfbaf6..3f47171 100644 --- a/cdx_toolkit/repackage/sources/athena.py +++ b/cdx_toolkit/repackage/sources/athena.py @@ -4,7 +4,7 @@ from typing import Iterator, Iterable, Optional from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.sources.base import RangeJobSource, CostEstimate +from cdx_toolkit.repackage.sources.base import RangeJobSource from cdx_toolkit.repackage.sources.sql_base import ( validate_result_columns, join_warc_url, @@ -28,7 +28,6 @@ def __init__( database: str, s3_output_location: str, warc_download_prefix: Optional[str], - n_crawls: Optional[int] = None, region_name: str = 'us-east-1', max_wait_time: int = 300, ): @@ -36,13 +35,9 @@ def __init__( self.database = database self.s3_output_location = s3_output_location self.warc_download_prefix = warc_download_prefix - self._n_crawls = n_crawls self.region_name = region_name self.max_wait_time = max_wait_time - def estimate_cost(self) -> CostEstimate: - return CostEstimate(n_crawls=self._n_crawls, engine='athena') - def _make_client(self): import boto3 from botocore.config import Config @@ -194,9 +189,3 @@ def iter_range_jobs(client, query_execution_id: str, warc_download_prefix: Optio validate_result_columns(list(first)) for row in itertools.chain([first], rows): yield row_to_range_job(row, warc_download_prefix) - - -def get_databases(client) -> list: - """Get list of available databases""" - response = client.list_databases(CatalogName='AwsDataCatalog') - return [db['Name'] for db in response['DatabaseList']] diff --git a/cdx_toolkit/repackage/sources/base.py b/cdx_toolkit/repackage/sources/base.py index e253cd7..faa4e4b 100644 --- a/cdx_toolkit/repackage/sources/base.py +++ b/cdx_toolkit/repackage/sources/base.py @@ -20,16 +20,11 @@ class RangeJobSource(ABC): A source owns its own stage-1 resource (Athena client, DuckDB connection, or an fsspec file handle) and yields RangeJobs synchronously; the pipeline orchestrator bridges the sync generator into the async fetch/write stages, and owns queueing, - the record limit, counting, and stop-sentinel emission.""" - - def estimate_cost(self) -> Optional[CostEstimate]: - """Return a CostEstimate for the cost guard, or None for sources that never - incur a per-scan charge (cdx files, csv).""" - return None + counting, and stop-sentinel emission.""" @abstractmethod def iter_range_jobs(self) -> Iterator[RangeJob]: """Yield RangeJobs. Implementations open their own client/connection lazily and close it in a finally. Each RangeJob carries `url` (authoritative for fetching) and, where known, the relative `filename`.""" - raise NotImplementedError + raise NotImplementedError # pragma: no cover diff --git a/cdx_toolkit/repackage/sources/duckdb.py b/cdx_toolkit/repackage/sources/duckdb.py index 86c0992..99e1e2e 100644 --- a/cdx_toolkit/repackage/sources/duckdb.py +++ b/cdx_toolkit/repackage/sources/duckdb.py @@ -2,9 +2,8 @@ from typing import Iterator, List, Optional from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.sources.base import RangeJobSource, CostEstimate +from cdx_toolkit.repackage.sources.base import RangeJobSource from cdx_toolkit.repackage.sources.sql_base import ( - build_sql, validate_result_columns, join_warc_url, REQUIRED_RESULT_COLUMNS, @@ -35,49 +34,16 @@ def _build_from_clause(index_path: str, crawls: Optional[List[str]]) -> str: class DuckDbSource(RangeJobSource): - """RangeJobs from a query against the CC columnar index via DuckDB (read_parquet on S3). - - Reads the public CommonCrawl parquet directly; AWS region/credentials come from - the environment (the public bucket is readable with valid credentials).""" - - def __init__( - self, - *, - query: Optional[str] = None, - hostnames: Optional[List[str]] = None, - domains: Optional[List[str]] = None, - crawls: Optional[List[str]] = None, - index_path: str, - warc_download_prefix: Optional[str], - limit: int = 0, - sort: bool = True, - region_name: str = 'us-east-1', - ): - self.raw_query = query - self.hostnames = hostnames - self.domains = domains - self.crawls = crawls - self.index_path = index_path + """RangeJobs (or rows) from a SQL query run with DuckDB, e.g. over the CC columnar index on S3. + + AWS region/credentials come from the environment (the AWS credential chain); the + public bucket is readable with valid credentials.""" + + def __init__(self, *, query: str, warc_download_prefix: Optional[str] = None, region_name: str = 'us-east-1'): + self.query = query self.warc_download_prefix = warc_download_prefix - self.limit = limit - self.sort = sort self.region_name = region_name - def estimate_cost(self) -> CostEstimate: - if self.raw_query is not None: - return CostEstimate(n_crawls=None, engine='duckdb') - return CostEstimate(n_crawls=len(self.crawls) if self.crawls else None, engine='duckdb') - - def _build_query(self) -> str: - if self.raw_query is not None: - return self.raw_query - from_clause = _build_from_clause(self.index_path, self.crawls) - # crawl pruning is done in the FROM glob, so no crawl IN (...) in the WHERE - return build_sql( - from_clause, self.hostnames, crawls=None, limit=self.limit, - url_host_registered_domains=self.domains, order_by=self.sort, - ) - def iter_rows(self, validate: bool = False) -> Iterator[dict]: """Run the query and yield the result rows as dicts (column name -> value).""" if not _HAS_DUCKDB: @@ -85,7 +51,7 @@ def iter_rows(self, validate: bool = False) -> Iterator[dict]: 'DuckDB engine requires optional dependencies. Install cdx_toolkit[duckdb].' ) - query = self._build_query() + query = self.query logger.info('Executing DuckDB query: %s', query) con = duckdb.connect() @@ -100,8 +66,12 @@ def iter_rows(self, validate: bool = False) -> Iterator[dict]: con.execute(f"CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION '{self.region_name}');") except Exception as e: # pragma: no cover - no credentials or aws extension unavailable logger.debug('duckdb AWS credential chain not used: %r', e) - # Be resilient to transient S3 read timeouts on large parquet partitions. - for stmt in ('SET http_timeout=120000;', 'SET http_retries=5;'): + # Be resilient to transient S3 read timeouts and throttling (503 SlowDown) on large parquet + # partitions: up to 8 retries, waiting 1s, 2s, 4s, ... (about 4 minutes in total) + for stmt in ( + 'SET http_timeout=120000;', 'SET http_retries=8;', 'SET http_retry_wait_ms=1000;', + 'SET http_retry_backoff=2;', + ): try: con.execute(stmt) except Exception as e: # pragma: no cover - setting unsupported on this duckdb diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py index f613e42..68ac172 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/warc_filter.py @@ -3,7 +3,6 @@ import logging import os import statistics -import sys from typing import Callable, List, Optional, Dict @@ -169,15 +168,8 @@ async def get_aws_clients(self) -> Optional[Dict]: Stage-1 clients/connections are owned by the source, so this only builds the read/write S3 clients used to fetch WARC ranges and write output. - - Raises: - SystemExit: If S3 is needed but Python version is < 3.9. """ if self.needs_aws(): - if sys.version_info.major < 3 or (sys.version_info.major >= 3 and sys.version_info.minor < 9): - logger.error('Reading and writing to S3 requires Python version >= 3.9') - sys.exit(1) - import aioboto3 from botocore.config import Config @@ -477,6 +469,7 @@ async def write_warc_records( writer = None current_file_sequence = 0 current_file_size = 0 + records_in_file = 0 new_writer_kwargs = dict( s3_client=s3_client, @@ -521,18 +514,24 @@ async def write_warc_records( writer, current_file_size = await self.start_file( current_file_sequence, **new_writer_kwargs ) - - # Check if we need to rotate files due to size limit - writer, current_file_sequence, current_file_size = await self.rotate_files( - writer=writer, - current_file_sequence=current_file_sequence, - current_file_size=current_file_size, - added_byte_size=len(item.data), - **new_writer_kwargs, - ) + records_in_file = 0 + elif records_in_file: + # Check if we need to rotate files due to size limit (a file gets at least + # one record, also if the record is larger than the size limit) + previous_sequence = current_file_sequence + writer, current_file_sequence, current_file_size = await self.rotate_files( + writer=writer, + current_file_sequence=current_file_sequence, + current_file_size=current_file_size, + added_byte_size=len(item.data), + **new_writer_kwargs, + ) + if current_file_sequence != previous_sequence: + records_in_file = 0 # Write actual response record await writer.write(item.data) + records_in_file += 1 current_file_size += len(item.data) tracker.add(bytes_count=len(item.data), records_count=item.job.records_count) diff --git a/tests/repackage/test_athena_source.py b/tests/repackage/test_athena_source.py new file mode 100644 index 0000000..a809adf --- /dev/null +++ b/tests/repackage/test_athena_source.py @@ -0,0 +1,137 @@ +"""AthenaSource and the Athena index backend with a fake boto3 Athena client (no AWS).""" +import json +import logging +from unittest.mock import patch + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.indexes import IndexQuery +from cdx_toolkit.indexes.sql import AthenaBackend +from cdx_toolkit.repackage.sources.athena import AthenaSource, report_query_cost, run_athena_query + +ATHENA = 'athena://athena.eu-west-1.amazonaws.com/mydb?S3OutputLocation=s3://bucket/athena/' +COLUMNS = ['url', 'fetch_time', 'fetch_status', 'warc_filename', 'warc_record_offset', 'warc_record_length'] +ROWS = [ + ['https://example.com/', '2024-07-13 10:22:33.000', '200', 'crawl-data/a.warc.gz', '10', '100'], + ['https://example.com/x', '2024-07-14 08:00:00.000', '404', 'crawl-data/b.warc.gz', '20', '200'], +] + + +class FakeAthenaClient: + """start_query_execution / get_query_execution (states over time) / paginated results.""" + + def __init__(self, states=('RUNNING', 'SUCCEEDED'), rows=ROWS, scanned=2 * 10 ** 9, reason=None): + self.states = list(states) + self.rows = rows + self.scanned = scanned + self.reason = reason + self.started = [] + self.stopped = [] + + def start_query_execution(self, **kwargs): + self.started.append(kwargs) + return {'QueryExecutionId': 'qid'} + + def get_query_execution(self, QueryExecutionId): + state = self.states.pop(0) if len(self.states) > 1 else self.states[0] + execution = {'Status': {'State': state}} + if self.reason: + execution['Status']['StateChangeReason'] = self.reason + if self.scanned is not None: + execution['Statistics'] = {'DataScannedInBytes': self.scanned} + return {'QueryExecution': execution} + + def stop_query_execution(self, QueryExecutionId): + self.stopped.append(QueryExecutionId) + + def get_paginator(self, name): + assert name == 'get_query_results' + header = {'Data': [{'VarCharValue': c} for c in COLUMNS]} + data = [{'Data': [{'VarCharValue': v} for v in row]} for row in self.rows] + # two pages: the header is only on the first one + pages = [{'ResultSet': {'Rows': [header] + data[:1]}}, {'ResultSet': {'Rows': data[1:]}}] + return type('Paginator', (), {'paginate': lambda self, **kwargs: iter(pages)})() + + +@pytest.fixture +def client(): + fake = FakeAthenaClient() + with patch.object(AthenaSource, '_make_client', return_value=fake), \ + patch('cdx_toolkit.repackage.sources.athena.time.sleep'): + yield fake + + +def test_source_rows_jobs_and_cost(client, caplog): + source = AthenaSource(query='SELECT 1', database='ccindex', s3_output_location='s3://b/p/', + warc_download_prefix='s3://commoncrawl') + + with caplog.at_level(logging.INFO): + rows = list(source.iter_rows()) + assert rows == [dict(zip(COLUMNS, row)) for row in ROWS] + assert client.started[0]['QueryExecutionContext'] == {'Database': 'ccindex'} + assert client.started[0]['ResultConfiguration'] == {'OutputLocation': 's3://b/p/'} + assert 'Athena scanned 2.00 GB, estimated cost ~$0.0100' in caplog.text + + jobs = list(source.iter_range_jobs()) + assert [(j.url, j.offset, j.length) for j in jobs] == [ + ('s3://commoncrawl/crawl-data/a.warc.gz', 10, 100), ('s3://commoncrawl/crawl-data/b.warc.gz', 20, 200), + ] + assert jobs[0].extra == { + 'url': 'https://example.com/', 'fetch_time': '2024-07-13 10:22:33.000', 'fetch_status': '200', + } + + +def test_index_athena_end_to_end(client, capsys): + main(['index', ATHENA, '--hostname', 'example.com', '--yes']) + + records = [json.loads(line) for line in capsys.readouterr().out.splitlines()] + assert records[0] == { + 'url': 'https://example.com/', 'timestamp': '20240713102233', 'status': '200', + 'filename': 'crawl-data/a.warc.gz', 'offset': '10', 'length': '100', + } + assert len(records) == 2 + assert 'FROM ccindex' in client.started[0]['QueryString'] + assert client.started[0]['QueryExecutionContext'] == {'Database': 'mydb'} + + +def test_athena_count(): + fake = FakeAthenaClient(rows=[['2']]) + fake.get_paginator = lambda name: type('P', (), {'paginate': lambda self, **kw: iter([ + {'ResultSet': {'Rows': [{'Data': [{'VarCharValue': 'n'}]}, {'Data': [{'VarCharValue': '42'}]}]}} + ])})() + with patch.object(AthenaSource, '_make_client', return_value=fake), \ + patch('cdx_toolkit.repackage.sources.athena.time.sleep'): + assert AthenaBackend(ATHENA, assume_yes=True).count(IndexQuery(domains=['example.com'])) == 42 + assert fake.started[0]['QueryString'].startswith('SELECT count(*) AS n FROM (') + + +def test_failed_query(caplog): + fake = FakeAthenaClient(states=['FAILED'], reason='SYNTAX_ERROR: line 1:8') + with caplog.at_level(logging.INFO), pytest.raises(Exception, match='Query failed with status: FAILED'): + run_athena_query(fake, 'SELECT', 'ccindex', 's3://b/p/') + assert 'SYNTAX_ERROR' in caplog.text + assert fake.stopped == [] + + +def test_query_timeout_is_cancelled(): + fake = FakeAthenaClient(states=['RUNNING']) + with patch('cdx_toolkit.repackage.sources.athena.time.sleep'), pytest.raises(TimeoutError): + run_athena_query(fake, 'SELECT 1', 'ccindex', 's3://b/p/', max_wait_time=0) + assert fake.stopped == ['qid'] + + +def test_cost_report_without_statistics(caplog): + with caplog.at_level(logging.INFO): + report_query_cost(FakeAthenaClient(scanned=None), 'qid') + assert 'Athena scanned' not in caplog.text + + +def test_make_client_uses_the_region(): + pytest.importorskip('boto3') + source = AthenaSource(query='SELECT 1', database='d', s3_output_location='s3://b/', warc_download_prefix=None, + region_name='eu-west-1') + with patch('boto3.client') as boto3_client: + source._make_client() + args, kwargs = boto3_client.call_args + assert args == ('athena',) and kwargs['config'].region_name == 'eu-west-1' diff --git a/tests/repackage/test_duckdb_source.py b/tests/repackage/test_duckdb_source.py new file mode 100644 index 0000000..d89710e --- /dev/null +++ b/tests/repackage/test_duckdb_source.py @@ -0,0 +1,68 @@ +"""DuckDbSource with a local parquet file, and its connection setup with a fake connection (no network).""" +from unittest.mock import MagicMock, patch + +import pytest + +from cdx_toolkit.repackage.sources import duckdb as duckdb_source +from cdx_toolkit.repackage.sources.duckdb import DuckDbSource, _build_from_clause + +from tests.conftest import requires_duckdb + + +def test_from_clause(): + assert _build_from_clause('s3://b/index/', ['CC-MAIN-2024-33']) == ( + "read_parquet(['s3://b/index/crawl=CC-MAIN-2024-33/subset=warc/*.parquet'], hive_partitioning=true)" + ) + assert _build_from_clause('s3://b/index', None) == ( + "read_parquet(['s3://b/index/crawl=*/subset=warc/*.parquet'], hive_partitioning=true)" + ) + + +@requires_duckdb +def test_range_jobs_from_parquet(tmp_path): + import duckdb + + path = tmp_path / 'index.parquet' + duckdb.connect().execute(f''' + COPY (SELECT * FROM (VALUES ('crawl-data/a.warc.gz', 10, 100, 'https://example.com/'), + ('crawl-data/b.warc.gz', 20, 200, 'https://example.com/x')) + AS t(warc_filename, warc_record_offset, warc_record_length, url)) + TO '{path}' (FORMAT parquet) + ''') + + source = DuckDbSource(query=f"SELECT * FROM read_parquet('{path}')", warc_download_prefix='s3://commoncrawl') + jobs = list(source.iter_range_jobs()) + assert [(j.url, j.offset, j.length, j.extra) for j in jobs] == [ + ('s3://commoncrawl/crawl-data/a.warc.gz', 10, 100, {'url': 'https://example.com/'}), + ('s3://commoncrawl/crawl-data/b.warc.gz', 20, 200, {'url': 'https://example.com/x'}), + ] + + with pytest.raises(ValueError, match='missing required columns'): + list(DuckDbSource(query=f"SELECT url FROM read_parquet('{path}')").iter_range_jobs()) + + +def test_without_duckdb(): + with patch.object(duckdb_source, '_HAS_DUCKDB', False), pytest.raises(RuntimeError, match=r'cdx_toolkit\[duckdb\]'): + list(DuckDbSource(query='SELECT 1').iter_rows()) + + +def test_connection_setup_and_hf_filesystem(): + con = MagicMock() + con.execute.return_value.description = [('n',)] + con.execute.return_value.fetchmany.side_effect = [[(1,)], []] + fake_duckdb = MagicMock(connect=MagicMock(return_value=con)) + hf_fs = MagicMock() + + fake_hf = MagicMock(HfFileSystem=MagicMock(return_value=hf_fs)) + query = "SELECT count(*) AS n FROM read_parquet('hf://buckets/x/y/*.parquet')" + with patch.object(duckdb_source, '_HAS_DUCKDB', True), \ + patch.object(duckdb_source, 'duckdb', fake_duckdb, create=True), \ + patch.dict('sys.modules', {'huggingface_hub': fake_hf}): + rows = list(DuckDbSource(query=query).iter_rows()) + + assert rows == [{'n': 1}] + statements = [c.args[0] for c in con.execute.call_args_list] + assert "SET TimeZone='UTC';" in statements + assert 'SET http_retries=8;' in statements and 'SET http_retry_wait_ms=1000;' in statements + con.register_filesystem.assert_called_once_with(hf_fs) + con.close.assert_called_once() diff --git a/tests/repackage/test_hf_utils.py b/tests/repackage/test_hf_utils.py index 1759d29..9b1de66 100644 --- a/tests/repackage/test_hf_utils.py +++ b/tests/repackage/test_hf_utils.py @@ -7,6 +7,7 @@ from cdx_toolkit.repackage.hf_utils import ( HfHttpReader, + make_hf_reader, signed_url_expiry, is_hf_url, hf_fs_path, @@ -186,3 +187,89 @@ async def no_sleep(_): assert reader.throttle_count == 2 assert reader.resolve_count == 1 + + +def test_resolve_without_redirect_reads_from_the_resolve_url(): + session = FakeSession(head_statuses=[200]) + reader = HfHttpReader(session, token='tok') + + assert asyncio.run(_read(reader, WARC_A, 0, 4)) == DATA[0:4] + url, headers = session.gets[0] + assert url.startswith('https://huggingface.co/buckets/') and headers['Authorization'] == 'Bearer tok' + + +def test_resolve_errors_are_retried_then_raised(): + # resolving is retried within each read attempt (3 x 3 attempts) + session = FakeSession(head_statuses=[500] * 3) + assert asyncio.run(_read(HfHttpReader(session), WARC_A, 0, 4)) == DATA[0:4] + + session = FakeSession(head_statuses=[500] * 9) + with pytest.raises(OSError, match='HF HEAD .* -> 500'): + asyncio.run(_read(HfHttpReader(session), WARC_A, 0, 4)) + assert len(session.heads) == 9 + + +@pytest.mark.parametrize( + 'get_statuses, offset, length, error', + [ + ([500, 500, 500], 0, 4, 'HF GET .* -> 500'), + ([], 1000, 100, 'returned 24 bytes, expected 100'), # short read + ], +) +def test_read_errors_are_retried_then_raised(get_statuses, offset, length, error): + session = FakeSession(get_statuses=get_statuses) + with pytest.raises(OSError, match=error): + asyncio.run(_read(HfHttpReader(session), WARC_A, offset, length)) + assert len(session.gets) == 3 + + +def test_cdn_rate_limit_is_waited_out(caplog): + session = FakeSession(get_statuses=[429]) + reader = HfHttpReader(session) + + async def run(): + data = await _read(reader, WARC_A, 0, 4) + await reader.aclose() + return data + + with caplog.at_level('INFO', logger='cdx_toolkit.repackage.hf_utils'): + assert asyncio.run(run()) == DATA[0:4] + assert reader.throttle_count == 1 + assert 'HF reader: 1 records read with 1 resolve requests' in caplog.text + assert 'HF resolver throttling: 1 x 429' in caplog.text + + +def test_gives_up_after_too_many_rate_limits(monkeypatch): + monkeypatch.setattr(HfHttpReader, 'MAX_THROTTLE_RETRIES', 0) + session = FakeSession(get_statuses=[429, 429, 429]) + with pytest.raises(OSError, match='HF 429: gave up after'): + asyncio.run(_read(HfHttpReader(session), WARC_A, 0, 4)) + + +def test_signed_url_cache_is_bounded(monkeypatch): + monkeypatch.setattr(HfHttpReader, 'MAX_CACHED_URLS', 1) + session = FakeSession() + reader = HfHttpReader(session) + + async def run(): + for url in (WARC_A, WARC_B, WARC_A): + await _read(reader, url, 0, 4) + + asyncio.run(run()) + assert len(session.heads) == 3 # WARC_A was evicted by WARC_B + assert list(reader._signed_urls) == [ + 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/crawl-data/a.warc.gz', + ] + + +def test_make_hf_reader(): + pytest.importorskip('aiohttp') + pytest.importorskip('huggingface_hub') + + async def run(): + with patch('huggingface_hub.get_token', return_value='tok'): + async with make_hf_reader() as reader: + return reader + + reader = asyncio.run(run()) + assert isinstance(reader, HfHttpReader) and reader.token == 'tok' diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py new file mode 100644 index 0000000..9f8d0ea --- /dev/null +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -0,0 +1,146 @@ +"""The WARC fetch/write pipeline end to end without network: fake HTTP and HF reads, in-process "processes".""" +import asyncio +import contextlib +import glob +import os +from io import BytesIO +from unittest.mock import Mock, patch + +import fsspec +import pytest +from warcio import WARCWriter +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.repackage import multiprocess +from cdx_toolkit.repackage.sources.records import RecordsSource +from cdx_toolkit.repackage.warc_filter import WARCFilter +from cdx_toolkit.repackage.warc_utils import get_bytes_from_warc_record + + +def warc_record_bytes(url): + record = WARCWriter(None).create_warc_record(url, 'resource', payload=BytesIO(b'content of ' + url.encode())) + return get_bytes_from_warc_record(record, gzip=True) + + +CAPTURES = [ + {'url': f'https://example.com/{i}', 'filename': f'crawl-data/{i % 2}.warc.gz', 'offset': str(i), 'length': '1'} + for i in range(4) +] + + +def fake_get(url, headers=None, **kwargs): + return Mock(content=warc_record_bytes(url + '#' + headers['Range'])) + + +def response_records(paths): + records = [] + for path in sorted(paths): + with fsspec.open(path, 'rb') as f: + records += [r for r in ArchiveIterator(f) if r.rec_type == 'resource'] + return records + + +def test_reader_and_writer_errors_are_logged_and_skipped(tmp_path, caplog): + calls = [] + + def flaky_get(url, headers=None, **kwargs): + calls.append(url) + if len(calls) == 1: + raise ConnectionError('network down') + return fake_get(url, headers) + + wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}) + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=flaky_get): + assert wf.filter() == 3 + + assert 'WARC Reader 0 failed' in caplog.text + assert len(response_records(glob.glob(str(tmp_path / 'out-*.warc.gz')))) == 3 + + +def test_writer_error_is_logged(tmp_path, caplog): + wf = WARCFilter(source=RecordsSource(CAPTURES[:1], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}) + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get), \ + patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header', side_effect=OSError('disk full')): + assert wf.filter() == 0 + assert 'WARC writer failed' in caplog.text + + +def test_hf_reads_use_the_hf_reader(tmp_path): + class FakeHfReader: + async def ranged_get(self, url, offset, length, **kwargs): + return warc_record_bytes(f'{url}#{offset}') + + @contextlib.asynccontextmanager + async def fake_make_hf_reader(): + yield FakeHfReader() + + wf = WARCFilter(source=RecordsSource(CAPTURES, 'hf://buckets/commoncrawl/commoncrawl'), + prefix_path=str(tmp_path / 'out'), writer_info={}, + warc_download_prefix='hf://buckets/commoncrawl/commoncrawl') + with patch('cdx_toolkit.repackage.warc_filter.make_hf_reader', fake_make_hf_reader): + assert wf.filter() == 4 + records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) + assert records[0].rec_headers.get_header('WARC-Target-URI').startswith('hf://buckets/commoncrawl/commoncrawl/') + + +@pytest.mark.parametrize('uvloop_installed', [True, False]) +def test_uvloop(monkeypatch, caplog, uvloop_installed): + monkeypatch.setenv('CDXT_UVLOOP', '1') + fake_uvloop = Mock(run=asyncio.run) if uvloop_installed else None + monkeypatch.setitem(__import__('sys').modules, 'uvloop', fake_uvloop) + wf = WARCFilter(source=RecordsSource([], None), prefix_path='/nonexistent/out', writer_info={}) + + with caplog.at_level('INFO'): + assert wf.filter() == 0 + assert ('Using uvloop event loop' if uvloop_installed else 'uvloop is not installed') in caplog.text + + +def test_no_aws_clients_without_s3(): + wf = WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, + warc_download_prefix='https://data.commoncrawl.org') + assert asyncio.run(wf.get_aws_clients()) is None + + +class InlineExecutor: + """ProcessPoolExecutor stand-in that runs the workers in this process (patches apply, coverage is measured).""" + + def __init__(self, max_workers, mp_context, initializer, initargs): + initializer(*initargs) + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def map(self, fn, items): + return [fn(item) for item in items] + + +@pytest.mark.parametrize('remote', [False, True]) +def test_multiprocess_repackage(tmp_path, remote): + prefix = 'memory://mp-test/out/RUN' if remote else str(tmp_path / 'RUN') + with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ + patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + n = multiprocess.run_multiprocess_repackage( + source=RecordsSource(CAPTURES, 'https://data.example.org'), n_processes=3, readers_per_process=2, + prefix_path=prefix, writer_info={'isPartOf': 'test'}, writer_subprefix='SUB', + write_paths_as_metadata_records=None, log_every_n=1, aws_region_name='us-east-1', max_attempts=1, + uvloop=False, max_file_size=1, + ) + + assert n == 4 + fs = fsspec.filesystem('memory') + if remote: + paths = ['memory://' + p for p in fs.glob('/mp-test/out/RUN-SUB-*.warc.gz')] + else: + paths = glob.glob(str(tmp_path / 'RUN-SUB-*.warc.gz')) + try: + # one file per record (--size 1), numbered by the shared counter + assert sorted(os.path.basename(p) for p in paths) == [f'RUN-SUB-00{i}.warc.gz' for i in range(1, 5)] + assert len(response_records(paths)) == 4 + finally: + if remote: + fs.rm('/mp-test', recursive=True) diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py index 6f79882..4798a94 100644 --- a/tests/unit/test_cdx_files.py +++ b/tests/unit/test_cdx_files.py @@ -107,7 +107,7 @@ def test_one_output_per_cdx_file(tmp_path, processes): '--processes', str(processes), ]) - outputs = sorted(str(p.relative_to(out)) for p in out.rglob('*') if p.is_file()) + outputs = sorted(p.relative_to(out).as_posix() for p in out.rglob('*') if p.is_file()) assert outputs == ['a/cdx-00001.gz', 'b/cdx-00002.gz'] with gzip.open(out / 'a' / 'cdx-00001.gz', 'rt') as f: # the lines are copied unchanged diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py new file mode 100644 index 0000000..6fe8995 --- /dev/null +++ b/tests/unit/test_usage_errors.py @@ -0,0 +1,134 @@ +"""Usage errors and small edge cases of `cdxt index` / `cdxt repackage` and their helpers.""" +import asyncio +import io +import runpy +import sys +from unittest.mock import patch + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.indexes import IndexQuery, UsageError +from cdx_toolkit.indexes.cdx_api import CdxApiBackend +from cdx_toolkit.indexes.cdx_files import CdxFilesBackend, load_url_patterns +from cdx_toolkit.indexes.patterns import domain_entry_to_surts, url_pattern_to_surts +from cdx_toolkit.indexes.sql import DuckDbBackend, sql_timestamp +from cdx_toolkit.record_io import RecordWriter, import_pyarrow, read_records, write_records +from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.sources.sql_base import quote_sql_string + +from tests.conftest import TEST_DATA_PATH + +CDX_PATH = str(TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz') + + +def test_python_m_cdx_toolkit_cli(monkeypatch, capsys): + monkeypatch.setattr(sys, 'argv', ['cdxt', '--version']) + with pytest.raises(SystemExit): + runpy.run_module('cdx_toolkit.cli', run_name='__main__') + assert capsys.readouterr().out.strip() + + +@pytest.mark.parametrize( + 'args, error', + [ + (['index', 'cc', '--url', 'example.com/*', '-o', 'out.txt'], 'cannot detect the output format of out.txt'), + (['index', CDX_PATH, '--domain', 'mnhn.fr', '--to', '2024-01'], 'invalid timestamp'), + (['index', CDX_PATH, '--url', 'ex*ample.com/*'], '* can only be at the start or the end'), + (['index', 'cc-url-index-s3', '--domain', 'example.com', '--to', '20241332'], "invalid timestamp '20241332'"), + (["index", "/data/it's.parquet", '--domain', 'example.com'], 'invalid index location'), + ], +) +def test_usage_errors(capsys, args, error): + with pytest.raises(SystemExit) as e: + main(args) + assert e.value.code == 2 + assert error in capsys.readouterr().err + + +@pytest.mark.parametrize( + 'command', [['index', 'cc', '--url', 'x.org/*', '-o', 'x.parquet'], ['repackage', 'x.parquet']] +) +def test_parquet_without_pyarrow(capsys, command): + with patch('cdx_toolkit.cli.index.import_pyarrow', side_effect=RuntimeError('needs pyarrow')), \ + patch('cdx_toolkit.cli.repackage.import_pyarrow', side_effect=RuntimeError('needs pyarrow')): + with pytest.raises(SystemExit): + main(command) + assert 'needs pyarrow' in capsys.readouterr().err + + +def test_import_pyarrow_missing(monkeypatch): + monkeypatch.setitem(sys.modules, 'pyarrow', None) + with pytest.raises(RuntimeError, match=r'cdx_toolkit\[parquet\]'): + import_pyarrow() + + +def test_cdx_api_dry_run_and_cc_sort(capsys): + main(['index', 'cc', '--domain', 'example.com', '--crawl', '3', '--limit', '5', '--dry-run']) + assert capsys.readouterr().out == 'cc: url=example.com limit=5 matchType=domain\ncrawls: 3\n' + + with patch('cdx_toolkit.CDXFetcher') as fetcher: + CdxApiBackend('cc', cc_sort='ascending')._fetcher(IndexQuery(urls=['x'])) + assert fetcher.call_args.kwargs == {'source': 'cc', 'cc_sort': 'ascending'} + + +def test_cdx_files_edge_cases(tmp_path, capsys): + with pytest.raises(UsageError, match='unknown --url-file-type'): + CdxFilesBackend(CDX_PATH).validate(IndexQuery(url_file='u.txt', url_file_type='regex')) + with pytest.raises(FileNotFoundError, match='no files found'): + CdxFilesBackend(str(tmp_path / 'cdx-*.gz')).files() + + # rows of a URL file without the column are skipped (after the first one) + path = tmp_path / 'urls.jsonl' + path.write_text('{"url": "example.com/a"}\n{"other": 1}\n{"url": "example.com/b"}\n') + assert load_url_patterns(str(path)) == ['example.com/a', 'example.com/b'] + + # the time range excludes all lines; it is shown in the dry run + assert CdxFilesBackend(CDX_PATH).count(IndexQuery(domains=['mnhn.fr'], to='2023')) == 0 + main(['index', CDX_PATH, '--domain', 'mnhn.fr', '--from', '2024', '--dry-run']) + assert 'timestamps: 20240101000000 to -' in capsys.readouterr().out + + +def test_pattern_errors(): + with pytest.raises(UsageError): + url_pattern_to_surts('ex*ample.com/*') + with pytest.raises(UsageError, match='no wildcards'): + domain_entry_to_surts('*.example.com') + + +def test_sql_helpers(): + with pytest.raises(UsageError): + sql_timestamp('20241332') + with pytest.raises(ValueError, match='control characters'): + quote_sql_string('a\nb') + with pytest.raises(UsageError): + DuckDbBackend("x'.parquet") + + +def test_record_io_unknown_formats(tmp_path): + with pytest.raises(ValueError, match='unknown format'): + RecordWriter(io.StringIO(), 'xml') + path = tmp_path / 'r.jsonl' + path.write_text('{"url": "x"}\n') + with pytest.raises(ValueError, match='unknown format'): + list(read_records(str(path), 'xml')) + + +def test_parquet_batches_exactly_full(tmp_path, monkeypatch): + pytest.importorskip('pyarrow') + monkeypatch.setattr('cdx_toolkit.record_io.PARQUET_BATCH_SIZE', 2) + path = str(tmp_path / 'r.parquet') + assert write_records(iter([{'url': str(i)} for i in range(4)]), path, 'parquet') == 4 + assert [r['url'] for r in read_records(path)] == ['0', '1', '2', '3'] + + +def test_hf_range_job_needs_a_reader(): + job = RangeJob(url='hf://buckets/ns/name/x.warc.gz', offset=0, length=1) + with pytest.raises(ValueError, match='requires an hf_reader'): + asyncio.run(job.ranged_get_bytes(1, 0)) + + class Reader: + async def ranged_get(self, url, offset, length, **kwargs): + return b'x' + + assert asyncio.run(job.ranged_get_bytes(1, 0, hf_reader=Reader())) == b'x' From 1293f6f4e464d4ca5c811d42204455726b0d3baf Mon Sep 17 00:00:00 2001 From: malteos Date: Sat, 10 Oct 2026 22:07:34 +0200 Subject: [PATCH 04/91] feat: Common Crawl's recommended AWS settings as defaults for S3 requests cdxt's S3 clients (the WARC fetcher's aioboto3 clients and s3fs for all other s3:// paths) use region us-east-1, retry_mode adaptive, max_attempts 100000, addressing_style virtual, payload_signing_enabled false and 2 concurrent requests (s3fs max_concurrency). The user stays in control: a default is only used for a setting that is not configured in the environment (AWS_REGION, AWS_DEFAULT_REGION, AWS_RETRY_MODE, AWS_MAX_ATTEMPTS), the AWS config file of the active profile or the fsspec configuration; CDXT_AWS_DEFAULTS=0 turns them off. The s3fs defaults are only set by the cdxt command (and its worker processes), not for library users. The multipart_threshold/multipart_chunksize settings are not applied: they are for whole-file transfers, cdxt reads WARC records with range requests and index files sequentially (and S3 parts are limited to 5 GiB). --- CHANGELOG.md | 2 + README.md | 11 ++ cdx_toolkit/aws.py | 127 +++++++++++++++++++++ cdx_toolkit/cli/__init__.py | 2 + cdx_toolkit/cli/repackage.py | 1 - cdx_toolkit/repackage/multiprocess.py | 4 +- cdx_toolkit/repackage/warc_filter.py | 34 ++---- tests/unit/test_aws.py | 156 ++++++++++++++++++++++++++ 8 files changed, 310 insertions(+), 27 deletions(-) create mode 100644 cdx_toolkit/aws.py create mode 100644 tests/unit/test_aws.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a31d2c..3fb084e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ a full URL in `warc_filename`/`filename` is used as it is (the `warc_url` column is gone) + fixed `--to` with a day (e.g. `--to 20240410`), which was padded to the end of the month, and February in leap years + removed the deprecated `cdx_iter` and `cdx_size` scripts + + S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual + addressing, no payload signing, 2 concurrent requests) unless configured by the user; `CDXT_AWS_DEFAULTS=0` turns them off + new extras: `athena`, `duckdb`, `hf`, `parquet`; `fsspec` and `aiofiles` are now required - 0.9.38 diff --git a/README.md b/README.md index 20a61e7..a07db1a 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,17 @@ $ pip install cdx_toolkit[parquet] # Parquet output (and input of cdxt repackage $ pip install cdx_toolkit[all] # install all optional dependencies ``` +### S3 settings + +For requests to S3 (e.g. `s3://commoncrawl`), `cdxt` uses Common Crawl's +recommended AWS settings: region `us-east-1`, `retry_mode = adaptive`, +`max_attempts = 100000`, and for S3 `addressing_style = virtual`, +`payload_signing_enabled = false` and `max_concurrent_requests = 2`. +Settings you configured yourself win: environment variables (`AWS_REGION`, +`AWS_DEFAULT_REGION`, `AWS_RETRY_MODE`, `AWS_MAX_ATTEMPTS`), the AWS config file +of your profile (`~/.aws/config`, `AWS_PROFILE`), and options in the fsspec +configuration. `CDXT_AWS_DEFAULTS=0` turns the defaults off. + ## Command-line tools The `cdxt` command line tool has two commands: `index` selects captures diff --git a/cdx_toolkit/aws.py b/cdx_toolkit/aws.py new file mode 100644 index 0000000..f93fd1f --- /dev/null +++ b/cdx_toolkit/aws.py @@ -0,0 +1,127 @@ +''' +Defaults for cdxt's requests to S3, as recommended by Common Crawl for downloading from s3://commoncrawl: + + region = us-east-1 + retry_mode = adaptive + max_attempts = 100000 + s3 = + addressing_style = virtual + payload_signing_enabled = false + max_concurrent_requests = 2 + (multipart_threshold = multipart_chunksize = 10GB: no multipart transfers) + +The user stays in control: a default is only used for a setting that is not configured in the +environment (AWS_REGION / AWS_DEFAULT_REGION, AWS_RETRY_MODE, AWS_MAX_ATTEMPTS) or in the AWS config +file of the active profile (~/.aws/config, AWS_PROFILE). CDXT_AWS_DEFAULTS=0 disables all defaults. + +The multipart settings are for whole-file transfers (`aws s3 cp`): cdxt reads WARC records with one +range request each and reads index files sequentially, so it makes no multipart downloads anyway. +''' +import functools +import logging +import os +from typing import Optional + + +LOGGER = logging.getLogger(__name__) + +DEFAULT_REGION = 'us-east-1' +DEFAULT_RETRY_MODE = 'adaptive' +DEFAULT_MAX_ATTEMPTS = 100000 # total attempts, as `max_attempts` in the AWS config file +DEFAULT_S3 = {'addressing_style': 'virtual', 'payload_signing_enabled': False} +DEFAULT_MAX_CONCURRENT_REQUESTS = 2 + +# environment variables that configure a setting (botocore reads them itself) +SETTING_ENV_VARS = { + 'region': ('AWS_REGION', 'AWS_DEFAULT_REGION'), + 'retry_mode': ('AWS_RETRY_MODE',), + 'max_attempts': ('AWS_MAX_ATTEMPTS',), +} + + +def defaults_enabled() -> bool: + return os.environ.get('CDXT_AWS_DEFAULTS', '1').strip().lower() not in ('0', 'false', 'no', 'off') + + +@functools.lru_cache(maxsize=None) +def user_aws_config() -> dict: + """The AWS config file settings of the active profile ({} without botocore or a config file).""" + try: + import botocore.session + + return dict(botocore.session.Session().get_scoped_config()) + except Exception as e: # botocore missing, unknown profile, unreadable config file + LOGGER.debug('no AWS config file settings: %r', e) + return {} + + +def user_configured(setting: str) -> bool: + if any(os.environ.get(name) for name in SETTING_ENV_VARS.get(setting, ())): + return True + return setting in user_aws_config() + + +def user_s3_setting(name: str): + s3 = user_aws_config().get('s3') + return s3.get(name) if isinstance(s3, dict) else None + + +def botocore_config_kwargs() -> dict: + """kwargs for botocore.config.Config with the defaults for the settings the user did not configure. + + botocore merges the `retries` and `s3` settings key by key with the environment and the AWS + config file, so the settings the user configured keep their values.""" + if not defaults_enabled(): + return {} + kwargs = {} + if not user_configured('region'): + kwargs['region_name'] = DEFAULT_REGION + retries = {} + if not user_configured('retry_mode'): + retries['mode'] = DEFAULT_RETRY_MODE + if not user_configured('max_attempts'): + retries['total_max_attempts'] = DEFAULT_MAX_ATTEMPTS + if retries: + kwargs['retries'] = retries + s3 = {name: value for name, value in DEFAULT_S3.items() if user_s3_setting(name) is None} + if s3: + kwargs['s3'] = s3 + return kwargs + + +def botocore_config(region_name: Optional[str] = None, **kwargs): + """A botocore Config with the defaults, plus the given (non-default) options, e.g. timeouts. + + An explicit `region_name` wins over the default and the user's configuration.""" + from botocore.config import Config + + config_kwargs = botocore_config_kwargs() + if region_name: + config_kwargs['region_name'] = region_name + return Config(**config_kwargs, **kwargs) + + +def max_concurrent_requests() -> int: + configured = user_s3_setting('max_concurrent_requests') + if configured is not None: + return int(configured) + return DEFAULT_MAX_CONCURRENT_REQUESTS + + +def configure_fsspec_s3() -> None: + """Use the defaults for s3:// paths opened with fsspec (s3fs), without changing options that are already + set in the fsspec configuration (~/.config/fsspec/, FSSPEC_S3_* environment variables).""" + if not defaults_enabled(): + return + import fsspec.config + + conf = fsspec.config.conf.setdefault('s3', {}) + config_kwargs = dict(conf.get('config_kwargs') or {}) + for key, value in botocore_config_kwargs().items(): + if isinstance(value, dict) and isinstance(config_kwargs.get(key), dict): + config_kwargs[key] = {**value, **config_kwargs[key]} + else: + config_kwargs.setdefault(key, value) + if config_kwargs: + conf['config_kwargs'] = config_kwargs + conf.setdefault('max_concurrency', max_concurrent_requests()) diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py index 85863a2..4612629 100644 --- a/cdx_toolkit/cli/__init__.py +++ b/cdx_toolkit/cli/__init__.py @@ -1,6 +1,7 @@ from argparse import ArgumentParser import sys +from cdx_toolkit.aws import configure_fsspec_s3 from cdx_toolkit.indexes import UsageError from cdx_toolkit.utils import get_version @@ -35,6 +36,7 @@ def main(args=None): parser = build_parser() cmd = parser.parse_args(args=args) set_loglevel(cmd.verbose) + configure_fsspec_s3() # Common Crawl's S3 defaults for s3:// paths, unless configured by the user cmdline = ' '.join(['cdxt'] + list(args)) try: cmd.func(cmd, cmdline) diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index e7a4446..848b50b 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -187,7 +187,6 @@ def repackage_by_warc_location(args: Namespace, records, info: dict) -> int: writer_subprefix=args.subprefix, write_paths_as_metadata_records=args.add_metadata_record, log_every_n=args.log_every_n, - aws_region_name='us-east-1', max_attempts=5, uvloop=os.environ.get('CDXT_UVLOOP') == '1', max_file_size=args.size, diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index 13474e1..5301d13 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -34,6 +34,7 @@ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from typing import Dict, List, Optional, Tuple +from cdx_toolkit.aws import configure_fsspec_s3 from cdx_toolkit.repackage.sources.csv import CsvSource, RangeJobCsvWriter from cdx_toolkit.repackage.warc_filter import WARCFilter from cdx_toolkit.repackage.warc_utils import generate_warc_filename @@ -61,6 +62,7 @@ def _output_filename(prefix_path: str, sequence: int, subprefix: Optional[str] = def _init_worker(file_sequence) -> None: global _FILE_SEQUENCE _FILE_SEQUENCE = file_sequence + configure_fsspec_s3() # a new process does not have the fsspec configuration of the main process def _next_file_sequence(_current: int) -> int: @@ -126,7 +128,7 @@ def run_multiprocess_repackage( writer_subprefix: Optional[str], write_paths_as_metadata_records: Optional[List[str]], log_every_n: int, - aws_region_name: str, + aws_region_name: Optional[str] = None, max_attempts: int, uvloop: bool, max_file_size: Optional[int] = 1_000_000_000, diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py index 68ac172..10a2214 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/warc_filter.py @@ -6,6 +6,7 @@ from typing import Callable, List, Optional, Dict +from cdx_toolkit.aws import botocore_config from cdx_toolkit.repackage.s3_utils import ( is_s3_url, ) @@ -53,7 +54,7 @@ def __init__( # writer_kwargs: Optional[Dict] = None, range_jobs_queue_size: int = 1000, warc_records_queue_size: int = 200, - aws_region_name: str = 'us-east-1', + aws_region_name: Optional[str] = None, warc_version: str = '1.0', content_type: Optional[str] = None, min_part_size: int = 5 * 1024 * 1024, # 5 MiB (for upload) @@ -76,7 +77,7 @@ def __init__( writer_kwargs: Optional additional kwargs for writers. range_jobs_queue_size: Maximum size of range jobs queue. warc_records_queue_size: Maximum size of WARC records queue. - aws_region_name: AWS region name for S3 operations. + aws_region_name: AWS region for S3 operations (default: the user's AWS configuration, else us-east-1). warc_version: WARC format version (e.g., '1.0' or '1.1'). content_type: Optional content type for WARC output. min_part_size: Minimum part byte size for multipart uploads (default: 5 MiB). @@ -145,24 +146,6 @@ def needs_aws(self) -> bool: """ return is_s3_url(self.warc_download_prefix) or is_s3_url(self.prefix_path) - def get_boto3_base_config(self) -> Dict: - """Get boto3 base configuration for AWS client. - - Returns: - Dict: Boto3 base configuration object with retry and timeout settings. - """ - # Calculate max connections based on parallelism - # Each reader + writer needs connections, plus some overhead for retries - # max_pool_connections = max(50, (self.num_readers + self.num_writers) * 2) - - return dict( - region_name=self.aws_region_name, - retries={ - 'max_attempts': max(2, self.max_attempts), - 'mode': 'adaptive', # Better than 'standard' for variable workloads - }, - ) - async def get_aws_clients(self) -> Optional[Dict]: """Return async S3 clients for WARC reads/writes if needed. @@ -171,24 +154,25 @@ async def get_aws_clients(self) -> Optional[Dict]: """ if self.needs_aws(): import aioboto3 - from botocore.config import Config session = aioboto3.Session() + # region, retries and S3 settings: Common Crawl's recommended defaults, unless the + # user configured them (see cdx_toolkit.aws); an explicit aws_region_name wins # High-throughput config for range reads - read_config = Config( + read_config = botocore_config( + region_name=self.aws_region_name, max_pool_connections=self.num_readers * 3, read_timeout=300, tcp_keepalive=True, - **self.get_boto3_base_config(), ) # Optimized config for multipart uploads (single writer) - write_config = Config( + write_config = botocore_config( + region_name=self.aws_region_name, max_pool_connections=8, read_timeout=120, connect_timeout=10, - **self.get_boto3_base_config(), ) return { diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py new file mode 100644 index 0000000..3a4533e --- /dev/null +++ b/tests/unit/test_aws.py @@ -0,0 +1,156 @@ +"""Common Crawl's S3 defaults (cdx_toolkit.aws): used for the settings the user did not configure.""" +import asyncio +from unittest.mock import patch + +import pytest + +from cdx_toolkit import aws + +FULL_DEFAULTS = { + 'region_name': 'us-east-1', + 'retries': {'mode': 'adaptive', 'total_max_attempts': 100000}, + 's3': {'addressing_style': 'virtual', 'payload_signing_enabled': False}, +} + + +@pytest.fixture +def no_user_config(monkeypatch, tmp_path): + for name in ('AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_RETRY_MODE', 'AWS_MAX_ATTEMPTS', 'CDXT_AWS_DEFAULTS', + 'AWS_PROFILE'): + monkeypatch.delenv(name, raising=False) + # botocore reads the AWS config file itself: use an empty one, not the one of the machine + monkeypatch.setenv('AWS_CONFIG_FILE', str(tmp_path / 'aws-config')) + with patch.object(aws, 'user_aws_config', return_value={}): + yield + + +def test_reads_the_aws_config_file(monkeypatch, tmp_path): + pytest.importorskip('botocore') + config_file = tmp_path / 'aws-config' + config_file.write_text( + '[default]\nmax_attempts = 5\ns3 =\n addressing_style = path\n max_concurrent_requests = 4\n' + ) + for name in ('AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_RETRY_MODE', 'AWS_MAX_ATTEMPTS', 'AWS_PROFILE'): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv('AWS_CONFIG_FILE', str(config_file)) + aws.user_aws_config.cache_clear() + try: + assert aws.botocore_config_kwargs() == { + 'region_name': 'us-east-1', + 'retries': {'mode': 'adaptive'}, + 's3': {'payload_signing_enabled': False}, + } + assert aws.max_concurrent_requests() == 4 + finally: + aws.user_aws_config.cache_clear() + + +def test_defaults(no_user_config): + assert aws.botocore_config_kwargs() == FULL_DEFAULTS + assert aws.max_concurrent_requests() == 2 + + +def test_environment_variables_win(no_user_config, monkeypatch): + monkeypatch.setenv('AWS_DEFAULT_REGION', 'eu-west-1') + monkeypatch.setenv('AWS_MAX_ATTEMPTS', '3') + assert aws.botocore_config_kwargs() == { + 'retries': {'mode': 'adaptive'}, + 's3': {'addressing_style': 'virtual', 'payload_signing_enabled': False}, + } + + +def test_aws_config_file_wins(no_user_config): + user_config = { + 'retry_mode': 'standard', 'max_attempts': '10', + 's3': {'addressing_style': 'path', 'max_concurrent_requests': '8'}, + } + with patch.object(aws, 'user_aws_config', return_value=user_config): + assert aws.botocore_config_kwargs() == {'region_name': 'us-east-1', 's3': {'payload_signing_enabled': False}} + assert aws.max_concurrent_requests() == 8 + + +def test_defaults_can_be_disabled(no_user_config, monkeypatch): + monkeypatch.setenv('CDXT_AWS_DEFAULTS', '0') + assert aws.botocore_config_kwargs() == {} + + +def test_botocore_resolves_the_defaults_and_the_user_settings(no_user_config, monkeypatch): + boto3 = pytest.importorskip('boto3') + + def client(**config): + # a new session reads the environment and the (empty) AWS config file + session = boto3.session.Session(aws_access_key_id='x', aws_secret_access_key='x') + return session.client('s3', config=aws.botocore_config(**config)) + + s3 = client() + assert s3.meta.region_name == 'us-east-1' + assert s3.meta.config.retries == {'mode': 'adaptive', 'total_max_attempts': 100000} + assert s3.meta.config.s3 == {'addressing_style': 'virtual', 'payload_signing_enabled': False} + + # a value from the environment is not overridden: botocore reads it itself + monkeypatch.setenv('AWS_MAX_ATTEMPTS', '3') + assert client().meta.config.retries == {'mode': 'adaptive', 'total_max_attempts': 3} + + # an explicit region wins + assert client(region_name='eu-central-1').meta.region_name == 'eu-central-1' + + +def test_configure_fsspec_s3_keeps_fsspec_settings(no_user_config, monkeypatch): + import fsspec.config + + user_conf = {'s3': {'config_kwargs': {'retries': {'mode': 'standard'}}, 'anon': False}} + monkeypatch.setattr(fsspec.config, 'conf', user_conf) + aws.configure_fsspec_s3() + assert fsspec.config.conf['s3'] == { + 'anon': False, + 'max_concurrency': 2, + 'config_kwargs': { + 'region_name': 'us-east-1', + 'retries': {'mode': 'standard', 'total_max_attempts': 100000}, + 's3': {'addressing_style': 'virtual', 'payload_signing_enabled': False}, + }, + } + + monkeypatch.setattr(fsspec.config, 'conf', {}) + monkeypatch.setenv('CDXT_AWS_DEFAULTS', 'off') + aws.configure_fsspec_s3() + assert fsspec.config.conf == {} + + +def test_s3fs_uses_the_defaults(no_user_config, monkeypatch): + s3fs = pytest.importorskip('s3fs') + import fsspec.config + + monkeypatch.setattr(fsspec.config, 'conf', {}) + aws.configure_fsspec_s3() + fs = s3fs.S3FileSystem(skip_instance_cache=True, key='x', secret='x') + assert fs.config_kwargs['retries'] == {'mode': 'adaptive', 'total_max_attempts': 100000} + assert fs.max_concurrency == 2 + + +def test_warc_fetcher_s3_clients(no_user_config): + pytest.importorskip('aioboto3') + from cdx_toolkit.repackage.sources.records import RecordsSource + from cdx_toolkit.repackage.warc_filter import WARCFilter + + wf = WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, + warc_download_prefix='s3://commoncrawl', n_parallel_readers=4) + with patch('aioboto3.Session') as session: + asyncio.run(wf.get_aws_clients()) + read_config, write_config = (c.kwargs['config'] for c in session.return_value.client.call_args_list) + for config in (read_config, write_config): + assert config.region_name == 'us-east-1' + assert config.retries == {'mode': 'adaptive', 'total_max_attempts': 100000} + assert config.s3 == {'addressing_style': 'virtual', 'payload_signing_enabled': False} + assert read_config.max_pool_connections == 12 and read_config.read_timeout == 300 + + +def test_cli_configures_fsspec(no_user_config, monkeypatch, tmp_path): + import fsspec.config + from cdx_toolkit.cli import main + + monkeypatch.setattr(fsspec.config, 'conf', {}) + captures = tmp_path / 'captures.jsonl' + captures.write_text('') + main(['repackage', str(captures), '--prefix', str(tmp_path / 'out')]) + assert fsspec.config.conf['s3']['config_kwargs']['retries']['total_max_attempts'] == 100000 From ab5e323a07d16729d87cd4f4fe74d740e7ffdccc Mon Sep 17 00:00:00 2001 From: malteos Date: Sat, 10 Oct 2026 22:17:20 +0200 Subject: [PATCH 05/91] feat: use the AWS defaults for Athena and DuckDB too - Athena client: the same botocore defaults (adaptive retries, 100000 attempts); the region of the athena:// location still wins - DuckDB (httpfs) on S3, as far as it has equivalents: region -> s3_region and the region of the credential secret, max_attempts -> http_retries (attempts - 1, with a constant 1s wait, because DuckDB's exponential backoff has no maximum), addressing style -> s3_url_style; settings the user configured (environment, AWS config file) are used instead, CDXT_AWS_DEFAULTS=0 keeps DuckDB's own defaults - fix DuckDB's http_timeout: it is in seconds, 120000 was ~33 hours, now 120s --- CHANGELOG.md | 3 +- README.md | 6 +++ cdx_toolkit/aws.py | 49 ++++++++++++++++++++++++- cdx_toolkit/repackage/sources/athena.py | 11 ++---- cdx_toolkit/repackage/sources/duckdb.py | 16 ++++---- tests/repackage/test_athena_source.py | 7 +++- tests/repackage/test_duckdb_source.py | 11 +++++- tests/unit/test_aws.py | 34 +++++++++++++++++ 8 files changed, 116 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fb084e..7c3de8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,8 @@ + fixed `--to` with a day (e.g. `--to 20240410`), which was padded to the end of the month, and February in leap years + removed the deprecated `cdx_iter` and `cdx_size` scripts + S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual - addressing, no payload signing, 2 concurrent requests) unless configured by the user; `CDXT_AWS_DEFAULTS=0` turns them off + addressing, no payload signing, 2 concurrent requests) unless configured by the user; `CDXT_AWS_DEFAULTS=0` turns them off; + also for Athena and, as far as DuckDB has equivalents, for DuckDB (fixes DuckDB's `http_timeout`, which is in seconds) + new extras: `athena`, `duckdb`, `hf`, `parquet`; `fsspec` and `aiofiles` are now required - 0.9.38 diff --git a/README.md b/README.md index a07db1a..d6b1aa6 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ Settings you configured yourself win: environment variables (`AWS_REGION`, of your profile (`~/.aws/config`, `AWS_PROFILE`), and options in the fsspec configuration. `CDXT_AWS_DEFAULTS=0` turns the defaults off. +The Athena client uses the same retry settings (its region is part of the +`athena://` location). DuckDB has no adaptive retry mode and no limit for +concurrent requests; for S3 it uses the region (`s3_region`), the addressing +style (`s3_url_style`) and `max_attempts` (`http_retries`, with 1 second between +retries). + ## Command-line tools The `cdxt` command line tool has two commands: `index` selects captures diff --git a/cdx_toolkit/aws.py b/cdx_toolkit/aws.py index f93fd1f..9d95f3d 100644 --- a/cdx_toolkit/aws.py +++ b/cdx_toolkit/aws.py @@ -16,11 +16,14 @@ The multipart settings are for whole-file transfers (`aws s3 cp`): cdxt reads WARC records with one range request each and reads index files sequentially, so it makes no multipart downloads anyway. + +The same settings are used for the Athena client and, as far as it has equivalents, for DuckDB. ''' import functools import logging import os -from typing import Optional +import re +from typing import List, Optional LOGGER = logging.getLogger(__name__) @@ -125,3 +128,47 @@ def configure_fsspec_s3() -> None: if config_kwargs: conf['config_kwargs'] = config_kwargs conf.setdefault('max_concurrency', max_concurrent_requests()) + + +def _user_value(setting: str) -> Optional[str]: + for name in SETTING_ENV_VARS.get(setting, ()): + if os.environ.get(name): + return os.environ[name] + value = user_aws_config().get(setting) + return str(value) if value is not None else None + + +def duckdb_settings(region_name: Optional[str] = None) -> List[str]: + """SET statements with the defaults for DuckDB's S3 access (httpfs), as far as DuckDB has equivalents. + + Settings the user configured (environment, AWS config file) are used instead of the defaults: + - region -> s3_region (an explicit `region_name` wins) + - max_attempts -> http_retries (attempts - 1), with a constant wait of 1s between retries, because + DuckDB's exponential backoff has no maximum wait + - s3 addressing_style -> s3_url_style (virtual: vhost, path: path) + There is no equivalent of retry_mode (DuckDB has no adaptive mode) and max_concurrent_requests + (only `threads`, which limits all of DuckDB's parallelism).""" + if not defaults_enabled(): + region = region_name or _user_value('region') + return [f"SET s3_region='{region}';"] if region and re.fullmatch(r'[a-z0-9-]+', region) else [] + region = region_name or _user_value('region') or DEFAULT_REGION + if not re.fullmatch(r'[a-z0-9-]+', region): + raise ValueError(f'invalid AWS region {region!r}') + attempts = _user_value('max_attempts') + attempts = int(attempts) if attempts and attempts.isdigit() else DEFAULT_MAX_ATTEMPTS + url_style = 'path' if (user_s3_setting('addressing_style') or DEFAULT_S3['addressing_style']) == 'path' else 'vhost' + return [ + f"SET s3_region='{region}';", + f"SET s3_url_style='{url_style}';", + f'SET http_retries={max(attempts - 1, 0)};', + 'SET http_retry_wait_ms=1000;', + 'SET http_retry_backoff=1;', + ] + + +def aws_region(region_name: Optional[str] = None) -> str: + """The region for AWS clients that need a concrete value (explicit, the user's, or the default).""" + region = region_name or _user_value('region') or DEFAULT_REGION + if not re.fullmatch(r'[a-z0-9-]+', region): + raise ValueError(f'invalid AWS region {region!r}') + return region diff --git a/cdx_toolkit/repackage/sources/athena.py b/cdx_toolkit/repackage/sources/athena.py index 3f47171..50c1be1 100644 --- a/cdx_toolkit/repackage/sources/athena.py +++ b/cdx_toolkit/repackage/sources/athena.py @@ -3,6 +3,7 @@ import time from typing import Iterator, Iterable, Optional +from cdx_toolkit.aws import botocore_config from cdx_toolkit.repackage.data_classes import RangeJob from cdx_toolkit.repackage.sources.base import RangeJobSource from cdx_toolkit.repackage.sources.sql_base import ( @@ -28,7 +29,7 @@ def __init__( database: str, s3_output_location: str, warc_download_prefix: Optional[str], - region_name: str = 'us-east-1', + region_name: Optional[str] = None, max_wait_time: int = 300, ): self.query = query @@ -40,13 +41,9 @@ def __init__( def _make_client(self): import boto3 - from botocore.config import Config - config = Config( - region_name=self.region_name, - read_timeout=60, - retries={'max_attempts': 3, 'mode': 'adaptive'}, - ) + # retries: Common Crawl's recommended AWS settings, unless configured by the user (see cdx_toolkit.aws) + config = botocore_config(region_name=self.region_name, read_timeout=60) return boto3.client('athena', config=config) def iter_rows(self) -> Iterator[dict]: diff --git a/cdx_toolkit/repackage/sources/duckdb.py b/cdx_toolkit/repackage/sources/duckdb.py index 99e1e2e..065fc8f 100644 --- a/cdx_toolkit/repackage/sources/duckdb.py +++ b/cdx_toolkit/repackage/sources/duckdb.py @@ -1,6 +1,7 @@ import logging from typing import Iterator, List, Optional +from cdx_toolkit.aws import aws_region, duckdb_settings from cdx_toolkit.repackage.data_classes import RangeJob from cdx_toolkit.repackage.sources.base import RangeJobSource from cdx_toolkit.repackage.sources.sql_base import ( @@ -39,7 +40,7 @@ class DuckDbSource(RangeJobSource): AWS region/credentials come from the environment (the AWS credential chain); the public bucket is readable with valid credentials.""" - def __init__(self, *, query: str, warc_download_prefix: Optional[str] = None, region_name: str = 'us-east-1'): + def __init__(self, *, query: str, warc_download_prefix: Optional[str] = None, region_name: Optional[str] = None): self.query = query self.warc_download_prefix = warc_download_prefix self.region_name = region_name @@ -57,21 +58,18 @@ def iter_rows(self, validate: bool = False) -> Iterator[dict]: con = duckdb.connect() try: con.execute('INSTALL httpfs; LOAD httpfs;') - con.execute(f"SET s3_region='{self.region_name}';") # CC timestamps are UTC (timestamp literals in queries, timestamps in the results) con.execute("SET TimeZone='UTC';") + region = aws_region(self.region_name) try: # find AWS credentials like boto3 does (env, ~/.aws profiles, SSO, instance roles) con.execute('INSTALL aws; LOAD aws;') - con.execute(f"CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION '{self.region_name}');") + con.execute(f"CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION '{region}');") except Exception as e: # pragma: no cover - no credentials or aws extension unavailable logger.debug('duckdb AWS credential chain not used: %r', e) - # Be resilient to transient S3 read timeouts and throttling (503 SlowDown) on large parquet - # partitions: up to 8 retries, waiting 1s, 2s, 4s, ... (about 4 minutes in total) - for stmt in ( - 'SET http_timeout=120000;', 'SET http_retries=8;', 'SET http_retry_wait_ms=1000;', - 'SET http_retry_backoff=2;', - ): + # region, retries and URL style: Common Crawl's recommended S3 settings, unless configured by + # the user (see cdx_toolkit.aws); a timeout of 2 minutes for large parquet partitions + for stmt in duckdb_settings(self.region_name) + ['SET http_timeout=120;']: try: con.execute(stmt) except Exception as e: # pragma: no cover - setting unsupported on this duckdb diff --git a/tests/repackage/test_athena_source.py b/tests/repackage/test_athena_source.py index a809adf..84f5395 100644 --- a/tests/repackage/test_athena_source.py +++ b/tests/repackage/test_athena_source.py @@ -127,11 +127,16 @@ def test_cost_report_without_statistics(caplog): assert 'Athena scanned' not in caplog.text -def test_make_client_uses_the_region(): +def test_make_client_uses_the_region(monkeypatch): pytest.importorskip('boto3') + monkeypatch.delenv('AWS_MAX_ATTEMPTS', raising=False) + monkeypatch.delenv('AWS_RETRY_MODE', raising=False) + monkeypatch.setattr('cdx_toolkit.aws.user_aws_config', lambda: {}) source = AthenaSource(query='SELECT 1', database='d', s3_output_location='s3://b/', warc_download_prefix=None, region_name='eu-west-1') with patch('boto3.client') as boto3_client: source._make_client() args, kwargs = boto3_client.call_args assert args == ('athena',) and kwargs['config'].region_name == 'eu-west-1' + # Common Crawl's recommended retry settings (cdx_toolkit.aws) + assert kwargs['config'].retries == {'mode': 'adaptive', 'total_max_attempts': 100000} diff --git a/tests/repackage/test_duckdb_source.py b/tests/repackage/test_duckdb_source.py index d89710e..9c934bb 100644 --- a/tests/repackage/test_duckdb_source.py +++ b/tests/repackage/test_duckdb_source.py @@ -46,7 +46,10 @@ def test_without_duckdb(): list(DuckDbSource(query='SELECT 1').iter_rows()) -def test_connection_setup_and_hf_filesystem(): +def test_connection_setup_and_hf_filesystem(monkeypatch): + for name in ('AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_MAX_ATTEMPTS', 'CDXT_AWS_DEFAULTS'): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr('cdx_toolkit.aws.user_aws_config', lambda: {}) con = MagicMock() con.execute.return_value.description = [('n',)] con.execute.return_value.fetchmany.side_effect = [[(1,)], []] @@ -63,6 +66,10 @@ def test_connection_setup_and_hf_filesystem(): assert rows == [{'n': 1}] statements = [c.args[0] for c in con.execute.call_args_list] assert "SET TimeZone='UTC';" in statements - assert 'SET http_retries=8;' in statements and 'SET http_retry_wait_ms=1000;' in statements + # Common Crawl's recommended S3 settings (cdx_toolkit.aws) and a timeout of 2 minutes + for statement in ("SET s3_region='us-east-1';", "SET s3_url_style='vhost';", 'SET http_retries=99999;', + 'SET http_retry_wait_ms=1000;', 'SET http_retry_backoff=1;', 'SET http_timeout=120;'): + assert statement in statements + assert "CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION 'us-east-1');" in statements con.register_filesystem.assert_called_once_with(hf_fs) con.close.assert_called_once() diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py index 3a4533e..99ceaea 100644 --- a/tests/unit/test_aws.py +++ b/tests/unit/test_aws.py @@ -154,3 +154,37 @@ def test_cli_configures_fsspec(no_user_config, monkeypatch, tmp_path): captures.write_text('') main(['repackage', str(captures), '--prefix', str(tmp_path / 'out')]) assert fsspec.config.conf['s3']['config_kwargs']['retries']['total_max_attempts'] == 100000 + + +def test_duckdb_settings(no_user_config): + assert aws.duckdb_settings() == [ + "SET s3_region='us-east-1';", + "SET s3_url_style='vhost';", + 'SET http_retries=99999;', + 'SET http_retry_wait_ms=1000;', + 'SET http_retry_backoff=1;', + ] + assert aws.duckdb_settings(region_name='eu-west-1')[0] == "SET s3_region='eu-west-1';" + + +def test_duckdb_settings_use_the_user_configuration(no_user_config, monkeypatch): + monkeypatch.setenv('AWS_REGION', 'eu-central-1') + monkeypatch.setenv('AWS_MAX_ATTEMPTS', '5') + with patch.object(aws, 'user_aws_config', return_value={'s3': {'addressing_style': 'path'}}): + assert aws.duckdb_settings()[:3] == [ + "SET s3_region='eu-central-1';", "SET s3_url_style='path';", 'SET http_retries=4;', + ] + assert aws.aws_region() == 'eu-central-1' + + monkeypatch.setenv('CDXT_AWS_DEFAULTS', '0') + assert aws.duckdb_settings() == ["SET s3_region='eu-central-1';"] # only what the user configured + monkeypatch.delenv('AWS_REGION') + assert aws.duckdb_settings() == [] + + +def test_invalid_region(no_user_config, monkeypatch): + monkeypatch.setenv('AWS_REGION', "us-east-1'; DROP") + with pytest.raises(ValueError, match='invalid AWS region'): + aws.aws_region() + with pytest.raises(ValueError, match='invalid AWS region'): + aws.duckdb_settings() From 45051fa004bcf47beadf78cae0c988294e7c301e Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:01:03 +0200 Subject: [PATCH 06/91] fix: abort the output file when writing fails, stop the pipeline cleanly - the S3 writer keeps a part's bytes until the part is uploaded; a part that cannot be uploaded aborts the multipart upload, and close() then raises instead of completing an object with missing bytes - any writer failure is fatal: the current file is aborted (local file removed, S3 upload aborted), the files written before stay complete - cancellation and failures cancel all tasks without waiting for queue space; the producer thread stops via an event instead of blocking on a full queue - the separate writer shutdown coordinator (a polling loop) is gone --- cdx_toolkit/repackage/local_writer.py | 22 +- cdx_toolkit/repackage/s3_writer.py | 62 +-- cdx_toolkit/repackage/warc_filter.py | 210 +++++----- cdx_toolkit/repackage/warc_utils.py | 8 + tests/repackage/test_producer.py | 58 +-- tests/repackage/test_s3_writer.py | 102 +++-- tests/repackage/test_warc_filter.py | 384 ------------------- tests/repackage/test_warc_filter_pipeline.py | 81 +++- 8 files changed, 315 insertions(+), 612 deletions(-) diff --git a/cdx_toolkit/repackage/local_writer.py b/cdx_toolkit/repackage/local_writer.py index 8fd9e48..69e365b 100644 --- a/cdx_toolkit/repackage/local_writer.py +++ b/cdx_toolkit/repackage/local_writer.py @@ -1,3 +1,6 @@ +import asyncio +import os + import aiofiles @@ -12,7 +15,13 @@ def __init__(self, file_path: str, buffer_size: int = 8192, mode: str = 'wb'): self.buffer = bytearray() async def start(self): - self.file_handle = await aiofiles.open(self.file_path, self.mode) + opening = asyncio.ensure_future(aiofiles.open(self.file_path, self.mode)) + try: + self.file_handle = await asyncio.shield(opening) + except asyncio.CancelledError: + # the file is created anyway: finish opening it, so that abort() can remove it + self.file_handle = await opening + raise async def write(self, data: bytes): self.buffer.extend(data) @@ -35,3 +44,14 @@ async def close(self): if self.file_handle: await self.file_handle.close() raise + + async def abort(self): + """Close and remove the (incomplete) file.""" + self.buffer.clear() + if self.file_handle: + await self.file_handle.close() + self.file_handle = None + try: + os.remove(self.file_path) + except FileNotFoundError: + pass diff --git a/cdx_toolkit/repackage/s3_writer.py b/cdx_toolkit/repackage/s3_writer.py index 124524d..ea82a65 100644 --- a/cdx_toolkit/repackage/s3_writer.py +++ b/cdx_toolkit/repackage/s3_writer.py @@ -82,7 +82,10 @@ async def mpu_abort(s3, bucket: str, key: str, upload_id: str): class S3ShardWriter: - """Manages one MPU: buffers bytes, uploads >=5 MiB parts, completes on close.""" + """Manages one MPU: buffers bytes, uploads >=5 MiB parts, completes on close. + + A part that cannot be uploaded (after the retries) makes the writer fail: the MPU is aborted, so no + object with missing bytes is created, and close() raises instead of completing the upload.""" def __init__( self, @@ -105,6 +108,7 @@ def __init__( self.part_number = 1 self.parts: List[Dict] = [] self.buffer = bytearray() + self.failed = False async def start(self): self.upload_id = await mpu_create( @@ -116,44 +120,39 @@ async def start(self): ) logger.info('Started MPU for %s (UploadId=%s)', self.shard_key, self.upload_id) - async def _flush_full_parts(self): - while len(self.buffer) >= self.min_part_size: - chunk = self.buffer[: self.min_part_size] - del self.buffer[: self.min_part_size] + async def _upload_part(self, size: int): + """Upload the first `size` bytes of the buffer as the next part; they leave the buffer only once uploaded.""" + if self.failed: + raise RuntimeError(f'the upload of {self.shard_key} failed before') + try: etag = await mpu_upload_part( self.s3_client, self.dest_bucket, self.shard_key, self.upload_id, self.part_number, - bytes(chunk), + bytes(self.buffer[:size]), self.max_attempts, self.base_backoff_seconds, ) - self.parts.append({'PartNumber': self.part_number, 'ETag': etag}) - self.part_number += 1 + except BaseException: + await self.abort() + raise + del self.buffer[:size] + self.parts.append({'PartNumber': self.part_number, 'ETag': etag}) + self.part_number += 1 async def write(self, data: bytes): self.buffer.extend(data) - await self._flush_full_parts() + while len(self.buffer) >= self.min_part_size: + await self._upload_part(self.min_part_size) async def close(self): + if self.failed: + raise RuntimeError(f'the upload of {self.shard_key} failed, it was aborted') + if self.buffer: + await self._upload_part(len(self.buffer)) try: - if self.buffer: - etag = await mpu_upload_part( - self.s3_client, - self.dest_bucket, - self.shard_key, - self.upload_id, - self.part_number, - bytes(self.buffer), - self.max_attempts, - self.base_backoff_seconds, - ) - self.parts.append({'PartNumber': self.part_number, 'ETag': etag}) - self.part_number += 1 - self.buffer.clear() - if self.parts: await mpu_complete( self.s3_client, @@ -165,8 +164,17 @@ async def close(self): self.base_backoff_seconds, ) logger.info('Completed MPU for %s with %d parts.', self.shard_key, len(self.parts)) - except Exception: + except BaseException: logger.exception('Completing MPU failed for %s; attempting abort.', self.shard_key) - if self.upload_id: - await mpu_abort(self.s3_client, self.dest_bucket, self.shard_key, self.upload_id) + await self.abort() raise + + async def abort(self): + """Abort the MPU (no object is created); the writer cannot be used afterwards.""" + if self.failed: + return + self.failed = True + self.buffer.clear() + if self.upload_id: + logger.warning('Aborting MPU for %s', self.shard_key) + await mpu_abort(self.s3_client, self.dest_bucket, self.shard_key, self.upload_id) diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py index 10a2214..e14f8a6 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/warc_filter.py @@ -1,8 +1,10 @@ import asyncio +import concurrent.futures import contextlib import logging import os import statistics +import threading from typing import Callable, List, Optional, Dict @@ -214,26 +216,33 @@ async def filter_async(self) -> int: hf_reader=hf_reader, ) - async def _produce_range_jobs(self, range_jobs_queue: asyncio.Queue) -> int: + async def _produce_range_jobs(self, range_jobs_queue: asyncio.Queue, stop: threading.Event) -> int: """Drive the (sync) source in a worker thread, feeding the async queue. - Owns counting and emitting one _STOP sentinel per reader in a finally -- so - readers never hang even if the source raises mid-iteration.""" + The thread waits for queue space in short steps, so it stops soon after `stop` is set (cancellation, + or a failed writer), also when the queue is full and nobody reads from it anymore.""" loop = asyncio.get_running_loop() - count = 0 def drain() -> int: - nonlocal count + count = 0 for job in self.source.iter_range_jobs(): - asyncio.run_coroutine_threadsafe(range_jobs_queue.put(job), loop).result() + future = asyncio.run_coroutine_threadsafe(range_jobs_queue.put(job), loop) + while True: + if stop.is_set(): + future.cancel() + return count + try: + future.result(timeout=0.1) + break + except concurrent.futures.TimeoutError: + pass count += 1 return count try: - await asyncio.to_thread(drain) + count = await asyncio.to_thread(drain) finally: - for _ in range(self.num_readers): - await range_jobs_queue.put(_STOP) + stop.set() # the thread keeps running if this task is cancelled; let it stop logger.info('Generated %d range jobs', count) return count @@ -248,6 +257,9 @@ async def _run_filter_pipeline( ) -> int: """Run the actual filter pipeline with or without S3 client. + A failing writer (or source) stops the whole pipeline, as does cancellation: all tasks are cancelled, + and the writer aborts the file it is writing (no incomplete WARC file or S3 object is left). + Args: range_jobs_queue: Queue for range jobs from the source. warc_records_queue: Queue for WARC record payloads. @@ -259,9 +271,10 @@ async def _run_filter_pipeline( int: Number of records written. """ logger.info('Starting job generator, %d WARC readers, 1 WARC writer', self.num_readers) + stop = threading.Event() # Generate range jobs from the configured source (bridged sync->async in a thread). - job_generators = asyncio.create_task(self._produce_range_jobs(range_jobs_queue)) + producer = asyncio.create_task(self._produce_range_jobs(range_jobs_queue, stop)) # Read WARC records based on file paths and ranges warc_readers = [ @@ -285,16 +298,29 @@ async def _run_filter_pipeline( ) ) - # Start writer coordination task - writer_coordinator = asyncio.create_task(self._coordinate_writer_shutdown(warc_readers, warc_records_queue)) - - await job_generators - logger.info('Range jobs submitted, monitoring readers and writer') + async def feed(): + """Wait for the jobs to be produced and read, then stop the readers and the writer.""" + await producer + for _ in warc_readers: + await range_jobs_queue.put(_STOP) + results = await asyncio.gather(*warc_readers) + await warc_records_queue.put(_STOP) + return results - # Wait for all tasks to complete - readers_results = await asyncio.gather(*warc_readers) - writer_result = await warc_writer - await writer_coordinator + feeder = asyncio.create_task(feed()) + tasks = [producer, *warc_readers, warc_writer, feeder] + try: + done, _ = await asyncio.wait({feeder, warc_writer}, return_when=asyncio.FIRST_EXCEPTION) + for task in done: + task.result() # raises the exception of a failed task + readers_results = await feeder + writer_result = await warc_writer + except BaseException: + stop.set() + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + raise readers_records = sum([result['stats']['total_records'] for result in readers_results]) readers_mb_per_sec = self.num_readers * statistics.mean( @@ -316,31 +342,6 @@ async def _run_filter_pipeline( return writer_stats['total_records'] - async def _coordinate_writer_shutdown(self, warc_readers: List[asyncio.Task], warc_records_queue: asyncio.Queue): - """Coordinate efficient shutdown of writers as readers complete. - - This prevents writers from waiting unnecessarily when all readers are done - and the records queue is being drained. - """ - completed_readers = 0 - - # Monitor reader completion - while completed_readers < len(warc_readers): - # Wait for any reader to complete - done, pending = await asyncio.wait( - warc_readers, - return_when=asyncio.FIRST_COMPLETED, - timeout=1.0, # Check periodically - ) - - if done: - completed_readers = len(warc_readers) - len(pending) - logger.debug(f'Readers completed: {completed_readers}/{len(warc_readers)}') - - # All readers completed - signal the writer to stop - logger.info('All readers completed, signaling writer to stop') - await warc_records_queue.put(_STOP) - async def read_warc_records( self, reader_id: int, @@ -403,8 +404,6 @@ async def read_warc_records( getattr(job, 'offset', -1), getattr(job, 'length', -1), ) - finally: - range_jobs_queue.task_done() return {'reader_id': reader_id, 'stats': tracker.get_stats()} @@ -472,67 +471,56 @@ async def write_warc_records( tracker.start() counter = 0 - # Response records + # Response records; any failure is fatal: the current file is aborted (removed, or its S3 upload + # aborted), the files written before stay complete try: while True: item = await warc_records_queue.get() - counter += 1 - try: - if item is _STOP: - stats = tracker.get_stats() - logger.info( - 'WARC writer stopping. Stats: %.1fs, %d items, %.1f MB written, %.2f MB/s write speed', - stats['elapsed'], - stats['total_requests'], - stats['total_bytes'] / (1024 * 1024), - stats['mb_per_sec'], - ) - should_stop = True - else: - should_stop = False - assert isinstance(item, RangePayload) - - if writer is None: - # first file, with header and metadata records - current_file_sequence = self.next_file_sequence(0) - writer, current_file_size = await self.start_file( - current_file_sequence, **new_writer_kwargs - ) - records_in_file = 0 - elif records_in_file: - # Check if we need to rotate files due to size limit (a file gets at least - # one record, also if the record is larger than the size limit) - previous_sequence = current_file_sequence - writer, current_file_sequence, current_file_size = await self.rotate_files( - writer=writer, - current_file_sequence=current_file_sequence, - current_file_size=current_file_size, - added_byte_size=len(item.data), - **new_writer_kwargs, - ) - if current_file_sequence != previous_sequence: - records_in_file = 0 - - # Write actual response record - await writer.write(item.data) - records_in_file += 1 - current_file_size += len(item.data) - tracker.add(bytes_count=len(item.data), records_count=item.job.records_count) - - # Log progress every N items - self.log_writer(counter=counter, tracker=tracker) - - except Exception: - logger.exception('WARC writer failed on %s', getattr(item, 'job', None)) - should_stop = False - finally: - warc_records_queue.task_done() - - if should_stop: + if item is _STOP: + stats = tracker.get_stats() + logger.info( + 'WARC writer stopping. Stats: %.1fs, %d items, %.1f MB written, %.2f MB/s write speed', + stats['elapsed'], + stats['total_requests'], + stats['total_bytes'] / (1024 * 1024), + stats['mb_per_sec'], + ) break - finally: + assert isinstance(item, RangePayload) + counter += 1 + + if writer is None: + # first file, with header and metadata records + current_file_sequence = self.next_file_sequence(0) + writer, current_file_size = await self.start_file(current_file_sequence, **new_writer_kwargs) + records_in_file = 0 + elif records_in_file and self.max_file_size and \ + current_file_size + len(item.data) > self.max_file_size: + # rotate due to the size limit (a file gets at least one record, also if the record is + # larger than the size limit); the finished file is not aborted if starting the next fails + finished, writer = writer, None + await finished.close() + current_file_sequence = self.next_file_sequence(current_file_sequence) + writer, current_file_size = await self.start_file(current_file_sequence, **new_writer_kwargs) + records_in_file = 0 + logger.info(f'Rotated to new WARC file sequence {current_file_sequence} due to size limit') + + # Write actual response record + await writer.write(item.data) + records_in_file += 1 + current_file_size += len(item.data) + tracker.add(bytes_count=len(item.data), records_count=item.job.records_count) + + # Log progress every N items + self.log_writer(counter=counter, tracker=tracker) + except BaseException: + logger.error('WARC writer failed, aborting the current WARC file') if writer is not None: - await writer.close() + await writer.abort() + raise + + if writer is not None: + await writer.close() return {'stats': tracker.get_stats()} @@ -560,23 +548,15 @@ def log_writer(self, counter: int, tracker: ThroughputTracker): stats['mb_per_sec'], ) - async def rotate_files( - self, writer, current_file_sequence: int, current_file_size: int, added_byte_size: int, **new_writer_kwargs - ): - """Check if we need to rotate files due to size limit and perform rotation if needed.""" - if self.max_file_size and current_file_size + added_byte_size > self.max_file_size: - await writer.close() - current_file_sequence = self.next_file_sequence(current_file_sequence) - writer, current_file_size = await self.start_file(current_file_sequence, **new_writer_kwargs) - logger.info(f'Rotated to new WARC file sequence {current_file_sequence} due to size limit') - - return writer, current_file_sequence, current_file_size - async def start_file(self, sequence: int, **new_writer_kwargs): """Start a new WARC file with its warcinfo record (and the metadata records, if any). Returns the writer and the number of bytes written.""" writer, file_size, warcinfo_id = await create_new_writer_with_header(sequence=sequence, **new_writer_kwargs) if self.write_paths_as_metadata_records: - file_size += await self.write_metadata_records(writer, warcinfo_id=warcinfo_id) + try: + file_size += await self.write_metadata_records(writer, warcinfo_id=warcinfo_id) + except BaseException: + await writer.abort() + raise return writer, file_size diff --git a/cdx_toolkit/repackage/warc_utils.py b/cdx_toolkit/repackage/warc_utils.py index 3f7a84d..566f7b2 100644 --- a/cdx_toolkit/repackage/warc_utils.py +++ b/cdx_toolkit/repackage/warc_utils.py @@ -133,6 +133,14 @@ async def create_new_writer_with_header( logger.debug('Initialzing new WARC writer for %s', warc_file_path) + try: + return await _start_with_header(new_writer, warc_file_path, writer_info, warc_version, gzip) + except BaseException: + await new_writer.abort() + raise + + +async def _start_with_header(new_writer, warc_file_path: str, writer_info: Dict, warc_version: str, gzip: bool): # Initialize writer await new_writer.start() diff --git a/tests/repackage/test_producer.py b/tests/repackage/test_producer.py index 2d6e6dc..f3e9fc3 100644 --- a/tests/repackage/test_producer.py +++ b/tests/repackage/test_producer.py @@ -1,50 +1,24 @@ -import asyncio +import glob +from unittest.mock import patch -from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.sources.base import RangeJobSource -from cdx_toolkit.repackage.warc_filter import WARCFilter, _STOP +import pytest +from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.warc_filter import WARCFilter -class FakeSource(RangeJobSource): - def __init__(self, jobs, raise_after=None): - self.jobs = jobs - self.raise_after = raise_after +class FailingSource: def iter_range_jobs(self): - for i, job in enumerate(self.jobs): - if self.raise_after is not None and i == self.raise_after: - raise RuntimeError('boom') - yield job - - -def _jobs(n): - return [ - RangeJob(url=f'https://data.commoncrawl.org/{i}.warc.gz', offset=i, length=1, filename=f'{i}.warc.gz') - for i in range(n) - ] - + yield RangeJob(url='https://data.commoncrawl.org/0.warc.gz', offset=0, length=1) + raise RuntimeError('boom') -def test_producer_emits_stops_even_when_source_raises(tmp_path): - """Regression: a source that raises mid-iteration must still release the readers.""" - wf = WARCFilter( - source=FakeSource(_jobs(2), raise_after=1), - prefix_path=str(tmp_path), - writer_info={}, - n_parallel_readers=3, - ) - async def run(): - queue: asyncio.Queue = asyncio.Queue() - try: - await wf._produce_range_jobs(queue) - except RuntimeError: - pass - items = [] - while not queue.empty(): - items.append(await queue.get()) - return items +def test_failing_source_fails_the_pipeline(tmp_path): + """A source that raises mid-iteration stops the readers and the writer, the error propagates.""" + async def read(self, *args, **kwargs): + return b'x' - items = asyncio.run(run()) - stops = sum(1 for it in items if it is _STOP) - assert wf.num_readers == 3 - assert stops == 3 + wf = WARCFilter(source=FailingSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=3) + with patch.object(RangeJob, 'ranged_get_bytes', read), pytest.raises(RuntimeError, match='boom'): + wf.filter() + assert glob.glob(str(tmp_path / 'out-*')) == [] diff --git a/tests/repackage/test_s3_writer.py b/tests/repackage/test_s3_writer.py index 38bfd3d..209109e 100644 --- a/tests/repackage/test_s3_writer.py +++ b/tests/repackage/test_s3_writer.py @@ -110,43 +110,6 @@ async def run_test(): asyncio.run(run_test()) -def test_shard_writer_flush_full_parts(): - """Test ShardWriter _flush_full_parts private method directly.""" - - async def run_test(): - with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part: - mock_upload_part.return_value = 'test-etag-flush' - - mock_s3 = AsyncMock() - writer = S3ShardWriter( - s3_client=mock_s3, - shard_key='test.warc.gz', - dest_bucket='test-bucket', - content_type='application/gzip', - min_part_size=50, # 50 bytes - max_attempts=3, - base_backoff_seconds=0.1, - ) - writer.upload_id = 'test-upload-id' - - # Pre-fill buffer with 150 bytes (should create 3 parts of 50 bytes each) - writer.buffer.extend(b'a' * 150) - - await writer._flush_full_parts() - - # Should have uploaded 3 full parts, no remainder - assert mock_upload_part.call_count == 3 - assert len(writer.parts) == 3 - assert writer.part_number == 4 # Next part would be #4 - assert len(writer.buffer) == 0 # All data flushed - - # Verify all parts were created correctly - for i in range(3): - assert writer.parts[i] == {'PartNumber': i + 1, 'ETag': 'test-etag-flush'} - - asyncio.run(run_test()) - - def test_shard_writer_close_with_buffer(): """Test ShardWriter close method with data remaining in buffer.""" @@ -309,3 +272,68 @@ async def run_test(): mock_s3.abort_multipart_upload.assert_called_once_with(Bucket=bucket, Key=key, UploadId=upload_id) asyncio.run(run_test()) + + +class FakeS3: + """upload_part fails `fail_times` times (ClientError), then stores the part bodies.""" + + def __init__(self, fail_times): + self.fail_times = fail_times + self.bodies = {} + self.completed = None + self.aborted = False + + async def upload_part(self, PartNumber, Body, **kwargs): + if self.fail_times: + self.fail_times -= 1 + from botocore.exceptions import ClientError + + raise ClientError({'Error': {'Code': '500', 'Message': 'boom'}}, 'UploadPart') + self.bodies[PartNumber] = Body + return {'ETag': f'etag-{PartNumber}'} + + async def complete_multipart_upload(self, MultipartUpload, **kwargs): + self.completed = b''.join(self.bodies[p['PartNumber']] for p in MultipartUpload['Parts']) + + async def abort_multipart_upload(self, **kwargs): + self.aborted = True + + +def make_writer(s3): + writer = S3ShardWriter(s3, 'out.warc.gz', 'bucket', None, min_part_size=4, max_attempts=2, + base_backoff_seconds=0) + writer.upload_id = 'upload-id' + return writer + + +def test_shard_writer_failed_part_aborts_the_upload(): + pytest.importorskip('botocore') + s3 = FakeS3(fail_times=2) # more than max_attempts + writer = make_writer(s3) + + async def run(): + with pytest.raises(Exception, match='boom'): + await writer.write(b'ABCD') + with pytest.raises(RuntimeError): + await writer.write(b'EFGH') + with pytest.raises(RuntimeError, match='aborted'): + await writer.close() + + asyncio.run(run()) + assert s3.aborted + assert s3.completed is None + + +def test_shard_writer_retried_part_keeps_every_byte_once(): + pytest.importorskip('botocore') + s3 = FakeS3(fail_times=1) # less than max_attempts + writer = make_writer(s3) + + async def run(): + await writer.write(b'ABCDEF') + await writer.write(b'GHIJ') + await writer.close() + + asyncio.run(run()) + assert s3.completed == b'ABCDEFGHIJ' + assert not s3.aborted diff --git a/tests/repackage/test_warc_filter.py b/tests/repackage/test_warc_filter.py index c22a5dc..39a8e21 100644 --- a/tests/repackage/test_warc_filter.py +++ b/tests/repackage/test_warc_filter.py @@ -1,6 +1,3 @@ -import asyncio - -from unittest.mock import AsyncMock, patch from cdx_toolkit.repackage.data_classes import ThroughputTracker from tests.conftest import TEST_DATA_PATH @@ -13,387 +10,6 @@ _FAKE_SOURCE = RecordsSource([], 'https://data.commoncrawl.org') -def test_filter_keyboard_interrupt_handling(caplog): - """Test that KeyboardInterrupt is properly handled in the filter method.""" - import logging - - # Set log level to capture WARNING messages - caplog.set_level(logging.WARNING, logger='cdx_toolkit.repackage.warc_filter') - - warc_filter = WARCFilter(source=_FAKE_SOURCE, prefix_path='/fake/prefix', writer_info={'writer_id': 1}) - - # Mock filter_async to raise KeyboardInterrupt - with patch.object(warc_filter, 'filter_async', side_effect=KeyboardInterrupt('Simulated user interrupt')): - # Call the filter method - result = warc_filter.filter() - - # Should return -1 when interrupted - assert result == -1 - - # Should log the warning message - assert 'Interrupted by user.' in caplog.text - - -def test_rotate_files_no_rotation_needed(): - """Test rotate_files when no rotation is needed (file size below limit).""" - - async def run_test(): - warc_filter = WARCFilter( - source=_FAKE_SOURCE, - prefix_path='/fake/prefix', - writer_info={'writer_id': 1}, - max_file_size=1000, # 1KB limit - ) - - mock_writer = AsyncMock() - current_file_sequence = 1 - current_file_size = 500 # 500 bytes - added_byte_size = 200 # Adding 200 bytes, total would be 700 (below limit) - - # Call rotate_files - result_writer, result_sequence, result_size = await warc_filter.rotate_files( - writer=mock_writer, - current_file_sequence=current_file_sequence, - current_file_size=current_file_size, - added_byte_size=added_byte_size, - output_path_prefix='/fake/output', - max_attempts=3, - base_backoff_seconds=1.0, - min_part_size=1024, - writer_info={'writer_id': 1}, - ) - - # Should return original values since no rotation occurred - assert result_writer == mock_writer - assert result_sequence == current_file_sequence - assert result_size == current_file_size - - # Writer should not be closed - mock_writer.close.assert_not_called() - - asyncio.run(run_test()) - - -def test_rotate_files_rotation_needed_without_resource_records(): - """Test rotate_files when rotation is needed and no resource records to write.""" - - async def run_test(): - warc_filter = WARCFilter( - source=_FAKE_SOURCE, - prefix_path='/fake/prefix', - writer_info={'writer_id': 1}, - max_file_size=1000, # 1KB limit - write_paths_as_metadata_records=None, # No resource records - ) - - mock_writer = AsyncMock() - mock_new_writer = AsyncMock() - current_file_sequence = 1 - current_file_size = 800 # 800 bytes - added_byte_size = 300 # Adding 300 bytes, total would be 1100 (above limit) - - # Mock create_new_writer_with_header - with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: - mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') # (writer, header_size, warcinfo_id) - - # Call rotate_files - result_writer, result_sequence, result_size = await warc_filter.rotate_files( - writer=mock_writer, - current_file_sequence=current_file_sequence, - current_file_size=current_file_size, - added_byte_size=added_byte_size, - output_path_prefix='/fake/output', - max_attempts=3, - base_backoff_seconds=1.0, - min_part_size=1024, - writer_info={'writer_id': 1}, - ) - - # Should have rotated - assert result_writer == mock_new_writer - assert result_sequence == current_file_sequence + 1 # Incremented - assert result_size == 150 # Header size only - - # Old writer should be closed - mock_writer.close.assert_called_once() - - # New writer should be created - mock_create.assert_called_once_with( - sequence=current_file_sequence + 1, - output_path_prefix='/fake/output', - max_attempts=3, - base_backoff_seconds=1.0, - min_part_size=1024, - writer_info={'writer_id': 1}, - ) - - asyncio.run(run_test()) - - -def test_rotate_files_rotation_needed_with_metadata_records(): - """Test rotate_files when rotation is needed and metadata records need to be written.""" - - async def run_test(): - warc_filter = WARCFilter( - source=_FAKE_SOURCE, - prefix_path='/fake/prefix', - writer_info={'writer_id': 1}, - max_file_size=1000, # 1KB limit - write_paths_as_metadata_records=['/fake/resource1.txt', '/fake/resource2.txt'], - ) - - mock_writer = AsyncMock() - mock_new_writer = AsyncMock() - current_file_sequence = 1 - current_file_size = 800 # 800 bytes - added_byte_size = 300 # Adding 300 bytes, total would be 1100 (above limit) - - # Mock create_new_writer_with_header - with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: - mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') - - # Mock write_metadata_records - with patch.object(warc_filter, 'write_metadata_records', return_value=75) as mock_write_resources: - # Call rotate_files - result_writer, result_sequence, result_size = await warc_filter.rotate_files( - writer=mock_writer, - current_file_sequence=current_file_sequence, - current_file_size=current_file_size, - added_byte_size=added_byte_size, - output_path_prefix='/fake/output', - max_attempts=3, - base_backoff_seconds=1.0, - min_part_size=1024, - writer_info={'writer_id': 1}, - ) - - # Should have rotated - assert result_writer == mock_new_writer - assert result_sequence == current_file_sequence + 1 - assert result_size == 150 + 75 # Header size + resource records size - - # Old writer should be closed - mock_writer.close.assert_called_once() - - # New writer should be created - mock_create.assert_called_once() - - # Resource records should be written - mock_write_resources.assert_called_once_with(mock_new_writer, warcinfo_id='warcinfo-123') - - asyncio.run(run_test()) - - -def test_rotate_files_no_max_file_size_set(): - """Test rotate_files when max_file_size is not set (None).""" - - async def run_test(): - warc_filter = WARCFilter( - source=_FAKE_SOURCE, - prefix_path='/fake/prefix', - writer_info={'writer_id': 1}, - max_file_size=None, # No limit - ) - - mock_writer = AsyncMock() - current_file_sequence = 1 - current_file_size = 999999999 # Very large file - added_byte_size = 999999999 # Very large addition - - # Call rotate_files - result_writer, result_sequence, result_size = await warc_filter.rotate_files( - writer=mock_writer, - current_file_sequence=current_file_sequence, - current_file_size=current_file_size, - added_byte_size=added_byte_size, - output_path_prefix='/fake/output', - max_attempts=3, - base_backoff_seconds=1.0, - min_part_size=1024, - writer_info={'writer_id': 1}, - ) - - # Should not rotate regardless of size - assert result_writer == mock_writer - assert result_sequence == current_file_sequence - assert result_size == current_file_size - - # Writer should not be closed - mock_writer.close.assert_not_called() - - asyncio.run(run_test()) - - -def test_rotate_files_edge_case_exact_limit(): - """Test rotate_files when the total size exactly equals the limit.""" - - async def run_test(): - warc_filter = WARCFilter( - source=_FAKE_SOURCE, - prefix_path='/fake/prefix', - writer_info={'writer_id': 1}, - max_file_size=1000, # 1KB limit - ) - - mock_writer = AsyncMock() - current_file_sequence = 1 - current_file_size = 700 # 700 bytes - added_byte_size = 300 # Adding 300 bytes, total would be exactly 1000 - - # Call rotate_files - result_writer, result_sequence, result_size = await warc_filter.rotate_files( - writer=mock_writer, - current_file_sequence=current_file_sequence, - current_file_size=current_file_size, - added_byte_size=added_byte_size, - output_path_prefix='/fake/output', - max_attempts=3, - base_backoff_seconds=1.0, - min_part_size=1024, - writer_info={'writer_id': 1}, - ) - - # Should not rotate when exactly at limit (only rotate when > limit) - assert result_writer == mock_writer - assert result_sequence == current_file_sequence - assert result_size == current_file_size - - # Writer should not be closed - mock_writer.close.assert_not_called() - - asyncio.run(run_test()) - - -def test_rotate_files_edge_case_just_over_limit(): - """Test rotate_files when the total size is just 1 byte over the limit.""" - - async def run_test(): - warc_filter = WARCFilter( - source=_FAKE_SOURCE, - prefix_path='/fake/prefix', - writer_info={'writer_id': 1}, - max_file_size=1000, # 1KB limit - ) - - mock_writer = AsyncMock() - mock_new_writer = AsyncMock() - current_file_sequence = 1 - current_file_size = 700 # 700 bytes - added_byte_size = 301 # Adding 301 bytes, total would be 1001 (1 byte over) - - # Mock create_new_writer_with_header - with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: - mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') - - # Call rotate_files - result_writer, result_sequence, result_size = await warc_filter.rotate_files( - writer=mock_writer, - current_file_sequence=current_file_sequence, - current_file_size=current_file_size, - added_byte_size=added_byte_size, - output_path_prefix='/fake/output', - max_attempts=3, - base_backoff_seconds=1.0, - min_part_size=1024, - writer_info={'writer_id': 1}, - ) - - # Should rotate when just over limit - assert result_writer == mock_new_writer - assert result_sequence == current_file_sequence + 1 - assert result_size == 150 - - # Old writer should be closed - mock_writer.close.assert_called_once() - - asyncio.run(run_test()) - - -def test_rotate_files_kwargs_passed_through(): - """Test that all kwargs are properly passed to create_new_writer_with_header.""" - - async def run_test(): - warc_filter = WARCFilter( - source=_FAKE_SOURCE, prefix_path='/fake/prefix', writer_info={'writer_id': 1}, max_file_size=1000 - ) - - mock_writer = AsyncMock() - mock_new_writer = AsyncMock() - - # Mock create_new_writer_with_header - with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: - mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') - - # Call rotate_files with various kwargs - await warc_filter.rotate_files( - writer=mock_writer, - current_file_sequence=1, - current_file_size=800, - added_byte_size=300, - output_path_prefix='/custom/output', - max_attempts=5, - base_backoff_seconds=2.5, - min_part_size=2048, - writer_info={'custom': 'info'}, - warc_version='1.1', - gzip=True, - custom_param='custom_value', - ) - - # Verify all kwargs are passed through - mock_create.assert_called_once_with( - sequence=2, # incremented from 1 - output_path_prefix='/custom/output', - max_attempts=5, - base_backoff_seconds=2.5, - min_part_size=2048, - writer_info={'custom': 'info'}, - warc_version='1.1', - gzip=True, - custom_param='custom_value', - ) - - asyncio.run(run_test()) - - -def test_rotate_files_logging(caplog): - """Test that rotation logs the appropriate message.""" - import logging - - async def run_test(): - # Set log level to capture INFO messages - caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.warc_filter') - - warc_filter = WARCFilter( - source=_FAKE_SOURCE, prefix_path='/fake/prefix', writer_info={'writer_id': 1}, max_file_size=1000 - ) - - mock_writer = AsyncMock() - mock_new_writer = AsyncMock() - - # Mock create_new_writer_with_header - with patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header') as mock_create: - mock_create.return_value = (mock_new_writer, 150, 'warcinfo-123') - - # Call rotate_files to trigger rotation - await warc_filter.rotate_files( - writer=mock_writer, - current_file_sequence=5, - current_file_size=800, - added_byte_size=300, - output_path_prefix='/fake/output', - max_attempts=3, - base_backoff_seconds=1.0, - min_part_size=1024, - writer_info={'writer_id': 1}, - ) - - # Check that the rotation log message was written - assert 'Rotated to new WARC file sequence 6 due to size limit' in caplog.text - - asyncio.run(run_test()) - - def test_log_writer(caplog): """Test log writer.""" import logging diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 9f8d0ea..0015068 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -2,7 +2,9 @@ import asyncio import contextlib import glob +import itertools import os +import threading from io import BytesIO from unittest.mock import Mock, patch @@ -12,6 +14,8 @@ from warcio.archiveiterator import ArchiveIterator from cdx_toolkit.repackage import multiprocess +from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.local_writer import LocalFileWriter from cdx_toolkit.repackage.sources.records import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter from cdx_toolkit.repackage.warc_utils import get_bytes_from_warc_record @@ -58,13 +62,78 @@ def flaky_get(url, headers=None, **kwargs): assert len(response_records(glob.glob(str(tmp_path / 'out-*.warc.gz')))) == 3 -def test_writer_error_is_logged(tmp_path, caplog): - wf = WARCFilter(source=RecordsSource(CAPTURES[:1], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), - writer_info={}) +def test_writer_error_stops_the_pipeline_and_removes_the_file(tmp_path): + wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}, max_file_size=1) + writes = [] + + original_write = LocalFileWriter.write + + async def failing_write(self, data): + writes.append(data) + if len(writes) == 4: # warcinfo + record of the first file, warcinfo + record of the second + raise OSError('disk full') + await original_write(self, data) + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get), \ - patch('cdx_toolkit.repackage.warc_filter.create_new_writer_with_header', side_effect=OSError('disk full')): - assert wf.filter() == 0 - assert 'WARC writer failed' in caplog.text + patch.object(LocalFileWriter, 'write', failing_write), pytest.raises(OSError, match='disk full'): + wf.filter() + # the first file is complete, the second was removed + assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == ['out-001.warc.gz'] + + +def test_rotation_by_size(tmp_path): + wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}, max_file_size=1) + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + assert wf.filter() == 4 + paths = sorted(glob.glob(str(tmp_path / 'out-*.warc.gz'))) + assert [os.path.basename(p) for p in paths] == [f'out-00{i}.warc.gz' for i in range(1, 5)] + assert all(len(response_records([p])) == 1 for p in paths) + + +def test_cancellation_with_a_full_queue_and_a_blocked_reader(tmp_path): + """Cancelling must not wait for queue space: the producer thread stops and the current file is removed.""" + source_done = threading.Event() + + class EndlessSource: + def iter_range_jobs(self): + try: + for i in itertools.count(): + yield RangeJob(url=f'https://data.example.org/{i}', offset=0, length=1) + finally: + source_done.set() + + reads = [] + + async def blocking_read(self, *args, **kwargs): + reads.append(self.url) + if len(reads) > 1: + await asyncio.Event().wait() # blocks forever + return b'x' + + wf = WARCFilter(source=EndlessSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, + range_jobs_queue_size=2) + + async def run(): + task = asyncio.create_task(wf.filter_async()) + while len(reads) < 2: + await asyncio.sleep(0.01) + await asyncio.sleep(0.2) # the queue fills up, the producer waits + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=2) + + def main(): + with patch.object(RangeJob, 'ranged_get_bytes', blocking_read): + asyncio.run(run()) + + thread = threading.Thread(target=main, daemon=True) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive(), 'shutdown did not finish' + assert source_done.wait(timeout=1), 'the producer thread is still waiting' + assert glob.glob(str(tmp_path / 'out-*')) == [] def test_hf_reads_use_the_hf_reader(tmp_path): From 72975f8355106e368952fb6357dfde820fd56a83 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:03:37 +0200 Subject: [PATCH 07/91] fix: cdxt repackage exits with status 1 when captures could not be fetched Captures that cannot be fetched are counted instead of only logged; the others are still written. The pipeline returns the requested, written and failed counts (summed over the worker processes), which the CLI logs; any failed capture makes it exit with status 1. The same applies to --wb. Ctrl-C exits with status 130. --- README.md | 7 +++ cdx_toolkit/cli/__init__.py | 6 ++- cdx_toolkit/cli/common.py | 5 ++ cdx_toolkit/cli/repackage.py | 31 +++++++---- cdx_toolkit/repackage/data_classes.py | 14 +++++ cdx_toolkit/repackage/multiprocess.py | 12 +++-- cdx_toolkit/repackage/warc_filter.py | 55 ++++++++------------ tests/repackage/test_warc_filter_pipeline.py | 16 +++--- tests/unit/test_cli.py | 45 +++++++++++++++- 9 files changed, 130 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index d6b1aa6..eb5afab 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,13 @@ file, `--creator`, `--operator`, `--is-part-of` and `--description` to fill in the `warcinfo` record, and `--add-metadata-record FILE` to add a file (e.g. the URL list of the job) as a metadata record to each WARC file. +A capture that cannot be fetched (after retries) does not stop the job: the +other captures are still written, and at the end `cdxt repackage` logs how +many captures were requested, written and failed, and exits with status 1. +A failure to write the output stops the job at once; the WARC file being +written is removed (or its S3 upload aborted), the files before it are +complete. + ### Migrating from the old commands `cdxt iter` is a deprecated alias of `cdxt index` with the same (new) diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py index 4612629..dbc0e66 100644 --- a/cdx_toolkit/cli/__init__.py +++ b/cdx_toolkit/cli/__init__.py @@ -6,7 +6,7 @@ from cdx_toolkit.utils import get_version from . import index, repackage -from .common import add_verbose_arg, set_loglevel +from .common import IncompleteError, add_verbose_arg, set_loglevel from .deprecated import DEPRECATED_COMMANDS, REMOVED_COMMANDS, exit_removed_command, get_command from .deprecated import rewrite_deprecated_command @@ -42,3 +42,7 @@ def main(args=None): cmd.func(cmd, cmdline) except (UsageError, FileNotFoundError) as e: parser.exit(2, f'cdxt {cmd.cmd}: error: {e}\n') + except IncompleteError as e: + parser.exit(1, f'cdxt {cmd.cmd}: error: {e}\n') + except KeyboardInterrupt: + parser.exit(130, f'cdxt {cmd.cmd}: interrupted\n') diff --git a/cdx_toolkit/cli/common.py b/cdx_toolkit/cli/common.py index 9052f19..7245e3b 100644 --- a/cdx_toolkit/cli/common.py +++ b/cdx_toolkit/cli/common.py @@ -6,6 +6,11 @@ LOGGER = logging.getLogger(__name__) + +class IncompleteError(RuntimeError): + """The command ran, but did not do all of its work, e.g. captures could not be fetched (exit status 1).""" + + SIZE_UNITS = { '': 1, 'b': 1, diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index 848b50b..1efd5d9 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -12,7 +12,7 @@ from cdx_toolkit.utils import get_version from cdx_toolkit.warc import fetch_wb_warc -from .common import add_verbose_arg, parse_size +from .common import IncompleteError, add_verbose_arg, parse_size LOGGER = logging.getLogger(__name__) @@ -153,14 +153,20 @@ def run(args: Namespace, cmdline: str): start_time = time.time() if args.wb: - n = repackage_from_wayback(args, records, info) + result = repackage_from_wayback(args, records, info) else: - n = repackage_by_warc_location(args, records, info) + result = repackage_by_warc_location(args, records, info) - LOGGER.info('wrote %d WARC records in %.1f seconds', n, time.time() - start_time) + LOGGER.info('wrote %d of %d captures to WARC files (%d failed) in %.1f seconds', + result.written, result.requested, result.failed, time.time() - start_time) + if result.failed: + raise IncompleteError( + f'{result.failed} of {result.requested} captures could not be fetched (see the log), ' + f'{result.written} were written' + ) -def repackage_by_warc_location(args: Namespace, records, info: dict) -> int: +def repackage_by_warc_location(args: Namespace, records, info: dict): """Copy the WARC records byte-for-byte with the async WARC fetcher (optionally multi-process).""" import fsspec @@ -210,7 +216,7 @@ def repackage_by_warc_location(args: Namespace, records, info: dict) -> int: raise UsageError(str(e)) -def repackage_from_wayback(args: Namespace, records, info: dict) -> int: +def repackage_from_wayback(args: Namespace, records, info: dict): """Replay the captures from a wayback machine, one by one, and write them with the WARC writer of cdx_toolkit.""" for option, used in ( ('--processes', args.processes > 1), @@ -220,18 +226,21 @@ def repackage_from_wayback(args: Namespace, records, info: dict) -> int: if used: raise UsageError(f'{option} cannot be used with --wb') + from cdx_toolkit.repackage.data_classes import RepackageResult + wb = WAYBACK_NAMES.get(args.wb, args.wb) - n = 0 + requested = written = 0 with cdx_toolkit.warc.get_writer(args.prefix, args.subprefix, info, size=args.size) as writer: for capture in records: + requested += 1 try: record = fetch_record(capture, wb) - except RuntimeError: # pragma: no cover - LOGGER.warning('skipping capture for RuntimeError 404: %s %s', capture['url'], capture['timestamp']) + except Exception as e: + LOGGER.error('cannot fetch capture %s %s: %r', capture.get('url'), capture.get('timestamp'), e) continue writer.write_record(record) - n += 1 - return n + written += 1 + return RepackageResult(requested=requested, written=written, failed=requested - written) def fetch_record(capture: Mapping, wb: Optional[str]): diff --git a/cdx_toolkit/repackage/data_classes.py b/cdx_toolkit/repackage/data_classes.py index 3465665..879e85c 100644 --- a/cdx_toolkit/repackage/data_classes.py +++ b/cdx_toolkit/repackage/data_classes.py @@ -40,6 +40,20 @@ def get_stats(self) -> dict: } +@dataclass(frozen=True) +class RepackageResult: + """Number of captures to repackage, written to WARC files, and failed (not fetched).""" + + requested: int = 0 + written: int = 0 + failed: int = 0 + + def __add__(self, other: 'RepackageResult') -> 'RepackageResult': + return RepackageResult( + self.requested + other.requested, self.written + other.written, self.failed + other.failed + ) + + @dataclass(frozen=True) class RangeJob: """Defines a S3 or HTTP range read request.""" diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index 5301d13..07690b0 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -35,6 +35,7 @@ from typing import Dict, List, Optional, Tuple from cdx_toolkit.aws import configure_fsspec_s3 +from cdx_toolkit.repackage.data_classes import RepackageResult from cdx_toolkit.repackage.sources.csv import CsvSource, RangeJobCsvWriter from cdx_toolkit.repackage.warc_filter import WARCFilter from cdx_toolkit.repackage.warc_utils import generate_warc_filename @@ -72,7 +73,7 @@ def _next_file_sequence(_current: int) -> int: return _FILE_SEQUENCE.value -def _run_worker(cfg: Dict) -> int: +def _run_worker(cfg: Dict) -> RepackageResult: """Worker entrypoint (runs in a child process): fetch one shard into the output WARCs.""" if cfg.get('uvloop'): os.environ['CDXT_UVLOOP'] = '1' @@ -133,9 +134,9 @@ def run_multiprocess_repackage( uvloop: bool, max_file_size: Optional[int] = 1_000_000_000, warc_download_prefix: Optional[str] = None, -) -> int: +) -> RepackageResult: """Shard -> N worker processes (rotating at --size) writing one global - [-]-NNN.warc.gz series. Returns the record count.""" + [-]-NNN.warc.gz series.""" tmp_dir = tempfile.mkdtemp(prefix='cdxt_shards_') start = time.time() try: @@ -176,7 +177,8 @@ def run_multiprocess_repackage( max_workers=len(configs), mp_context=mp_context, initializer=_init_worker, initargs=(file_sequence,) ) as ex: results = list(ex.map(_run_worker, configs)) - total = sum(r for r in results if r and r > 0) + result = sum(results, RepackageResult()) + total = result.written n_files = file_sequence.value fetch_elapsed = time.time() - start logger.info('All %d workers done: %d records in %d files in %.1fs (%.0f rec/s)', @@ -188,7 +190,7 @@ def run_multiprocess_repackage( logger.info('Repackage complete -> %d files %s (%.1fs total)', n_files, _output_filename(prefix_path, sequence=1, subprefix=writer_subprefix), time.time() - start) - return total + return result finally: shutil.rmtree(tmp_dir, ignore_errors=True) diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py index e14f8a6..707511b 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/warc_filter.py @@ -13,7 +13,7 @@ is_s3_url, ) from cdx_toolkit.repackage.hf_utils import is_hf_url, make_hf_reader -from cdx_toolkit.repackage.data_classes import RangeJob, RangePayload, ThroughputTracker +from cdx_toolkit.repackage.data_classes import RangeJob, RangePayload, RepackageResult, ThroughputTracker from cdx_toolkit.repackage.warc_utils import create_new_writer_with_header from cdx_toolkit.repackage.sources.base import RangeJobSource from cdx_toolkit.repackage.warc_utils import get_bytes_from_warc_record, get_metadata_record_from_path @@ -118,11 +118,11 @@ def __init__( # multi-process mode shares one counter across processes, so file names are final right away. self.next_file_sequence = next_file_sequence or (lambda sequence: sequence + 1) - def filter(self) -> int: + def filter(self) -> RepackageResult: """Perform the filtering process (calls async method via asyncio.run). - Returns: - int: Number of records written, or -1 if interrupted. + Captures that cannot be fetched are counted as failed (the others are still written); a failure to + write raises. """ runner = asyncio.run if os.environ.get('CDXT_UVLOOP') == '1': @@ -133,12 +133,7 @@ def filter(self) -> int: logger.info('Using uvloop event loop') except ImportError: logger.warning('CDXT_UVLOOP=1 but uvloop is not installed; using default asyncio loop') - try: - return runner(self.filter_async()) - except KeyboardInterrupt: - logger.warning('Interrupted by user.') - - return -1 + return runner(self.filter_async()) def needs_aws(self) -> bool: """Returns true if the read/write (stage 2/3) S3 clients are needed. @@ -184,12 +179,8 @@ async def get_aws_clients(self) -> Optional[Dict]: else: return None - async def filter_async(self) -> int: - """Filter process using a three stage approach (job generator, warc reader, warc writer). - - Returns: - int: Number of records written. - """ + async def filter_async(self) -> RepackageResult: + """Filter process using a three stage approach (job generator, warc reader, warc writer).""" range_jobs_queue: asyncio.Queue = asyncio.Queue(maxsize=self.range_jobs_queue_size) warc_records_queue: asyncio.Queue = asyncio.Queue(maxsize=self.warc_records_queue_size) @@ -254,7 +245,7 @@ async def _run_filter_pipeline( read_s3_client=None, write_s3_client=None, hf_reader=None, - ) -> int: + ) -> RepackageResult: """Run the actual filter pipeline with or without S3 client. A failing writer (or source) stops the whole pipeline, as does cancellation: all tasks are cancelled, @@ -266,9 +257,6 @@ async def _run_filter_pipeline( read_s3_client: Optional S3 client for reads from S3. write_s3_client: Optional S3 client for writes S3. hf_reader: Optional HF reader for hf:// downloads. - - Returns: - int: Number of records written. """ logger.info('Starting job generator, %d WARC readers, 1 WARC writer', self.num_readers) stop = threading.Event() @@ -300,12 +288,12 @@ async def _run_filter_pipeline( async def feed(): """Wait for the jobs to be produced and read, then stop the readers and the writer.""" - await producer + requested = await producer for _ in warc_readers: await range_jobs_queue.put(_STOP) results = await asyncio.gather(*warc_readers) await warc_records_queue.put(_STOP) - return results + return requested, results feeder = asyncio.create_task(feed()) tasks = [producer, *warc_readers, warc_writer, feeder] @@ -313,7 +301,7 @@ async def feed(): done, _ = await asyncio.wait({feeder, warc_writer}, return_when=asyncio.FIRST_EXCEPTION) for task in done: task.result() # raises the exception of a failed task - readers_results = await feeder + requested, readers_results = await feeder writer_result = await warc_writer except BaseException: stop.set() @@ -340,7 +328,10 @@ async def feed(): f"{writer_stats['records_per_sec']:.2f} rec/s" ) - return writer_stats['total_records'] + failed = sum(result['failed'] for result in readers_results) + if failed: + logger.error('%d of %d captures could not be fetched', failed, requested) + return RepackageResult(requested=requested, written=writer_stats['total_records'], failed=failed) async def read_warc_records( self, @@ -365,6 +356,7 @@ async def read_warc_records( tracker = ThroughputTracker() tracker.start() counter = 0 + failed = 0 while True: job = await range_jobs_queue.get() @@ -395,17 +387,12 @@ async def read_warc_records( self.log_reader(reader_id=reader_id, counter=counter, tracker=tracker) await warc_records_queue.put(RangePayload(job=job, data=data)) - except Exception: - logger.exception( - 'WARC Reader %d failed on %s/%s [%d,%d]', - reader_id, - getattr(job, 'bucket', '?'), - getattr(job, 'key', '?'), - getattr(job, 'offset', -1), - getattr(job, 'length', -1), - ) + except Exception as e: + # the capture is lost, the others are still fetched; the run fails at the end + failed += 1 + logger.error('WARC Reader %d failed on %s [%d,%d]: %r', reader_id, job.url, job.offset, job.length, e) - return {'reader_id': reader_id, 'stats': tracker.get_stats()} + return {'reader_id': reader_id, 'stats': tracker.get_stats(), 'failed': failed} async def write_metadata_records(self, writer, warcinfo_id: str) -> int: """Write WARC metadata records based on paths""" diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 0015068..af3ea13 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -14,7 +14,7 @@ from warcio.archiveiterator import ArchiveIterator from cdx_toolkit.repackage import multiprocess -from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.data_classes import RangeJob, RepackageResult from cdx_toolkit.repackage.local_writer import LocalFileWriter from cdx_toolkit.repackage.sources.records import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter @@ -44,7 +44,7 @@ def response_records(paths): return records -def test_reader_and_writer_errors_are_logged_and_skipped(tmp_path, caplog): +def test_failed_captures_are_counted_and_the_others_written(tmp_path): calls = [] def flaky_get(url, headers=None, **kwargs): @@ -56,9 +56,7 @@ def flaky_get(url, headers=None, **kwargs): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}) with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=flaky_get): - assert wf.filter() == 3 - - assert 'WARC Reader 0 failed' in caplog.text + assert wf.filter() == RepackageResult(requested=4, written=3, failed=1) assert len(response_records(glob.glob(str(tmp_path / 'out-*.warc.gz')))) == 3 @@ -86,7 +84,7 @@ def test_rotation_by_size(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, max_file_size=1) with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): - assert wf.filter() == 4 + assert wf.filter().written == 4 paths = sorted(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert [os.path.basename(p) for p in paths] == [f'out-00{i}.warc.gz' for i in range(1, 5)] assert all(len(response_records([p])) == 1 for p in paths) @@ -149,7 +147,7 @@ async def fake_make_hf_reader(): prefix_path=str(tmp_path / 'out'), writer_info={}, warc_download_prefix='hf://buckets/commoncrawl/commoncrawl') with patch('cdx_toolkit.repackage.warc_filter.make_hf_reader', fake_make_hf_reader): - assert wf.filter() == 4 + assert wf.filter().written == 4 records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert records[0].rec_headers.get_header('WARC-Target-URI').startswith('hf://buckets/commoncrawl/commoncrawl/') @@ -162,7 +160,7 @@ def test_uvloop(monkeypatch, caplog, uvloop_installed): wf = WARCFilter(source=RecordsSource([], None), prefix_path='/nonexistent/out', writer_info={}) with caplog.at_level('INFO'): - assert wf.filter() == 0 + assert wf.filter() == RepackageResult() assert ('Using uvloop event loop' if uvloop_installed else 'uvloop is not installed') in caplog.text @@ -200,7 +198,7 @@ def test_multiprocess_repackage(tmp_path, remote): uvloop=False, max_file_size=1, ) - assert n == 4 + assert n == RepackageResult(requested=4, written=4) fs = fsspec.filesystem('memory') if remote: paths = ['memory://' + p for p in fs.glob('/mp-test/out/RUN-SUB-*.warc.gz')] diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 1b48ec1..dc2bf6c 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -12,6 +12,7 @@ from cdx_toolkit.cli import main from cdx_toolkit.cli.common import parse_size from cdx_toolkit.cli.deprecated import get_command, rewrite_deprecated_command, translate +from cdx_toolkit.repackage.data_classes import RepackageResult import cdx_toolkit.cli.index @@ -123,6 +124,7 @@ def test_repackage_options_are_passed_to_the_warc_fetcher(tmp_path): captures = _write_captures(tmp_path / 'captures.jsonl', []) with patch('cdx_toolkit.repackage.warc_filter.WARCFilter') as warc_filter: + warc_filter.return_value.filter.return_value = RepackageResult() main([ 'repackage', captures, '--prefix', 'FOO', '--subprefix', 'BAR', '--size', '1MiB', '--creator', 'me', '--description', 'test', '--add-metadata-record', 'a.txt', @@ -145,7 +147,8 @@ def test_repackage_options_are_passed_to_the_warc_fetcher(tmp_path): def test_repackage_with_processes_uses_multiprocess(tmp_path): captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) - with patch('cdx_toolkit.repackage.multiprocess.run_multiprocess_repackage', return_value=1) as run: + result = RepackageResult(1, 1) + with patch('cdx_toolkit.repackage.multiprocess.run_multiprocess_repackage', return_value=result) as run: main(['repackage', captures, '--prefix', str(tmp_path / 'out'), '--processes', '3', '--subprefix', 'SUB']) kwargs = run.call_args.kwargs @@ -244,3 +247,43 @@ def test_verbose_before_and_after_subcommand(tmp_path): assert set_loglevel.call_args.args == (1,) main(['repackage', '-vv', captures, '--prefix', str(tmp_path / 'out')]) assert set_loglevel.call_args.args == (2,) + + +@pytest.mark.parametrize('n_failing', [1, 2]) +def test_repackage_exits_1_if_captures_could_not_be_fetched(tmp_path, capsys, n_failing): + captures = [dict(CC_CAPTURE, offset=str(i)) for i in range(2)] + path = _write_captures(tmp_path / 'captures.jsonl', captures) + calls = [] + + def get(url, headers=None, **kwargs): + calls.append(url) + if len(calls) <= n_failing: + raise ConnectionError('network down') + return Mock(content=_gzipped_warc_record()) + + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=get), \ + pytest.raises(SystemExit) as exc: + main(['repackage', path, '--prefix', str(tmp_path / 'out')]) + assert exc.value.code == 1 + assert f'{n_failing} of 2 captures could not be fetched' in capsys.readouterr().err + assert len(list(tmp_path.glob('out-*.warc.gz'))) == (1 if n_failing == 1 else 0) + + +def test_repackage_fails_if_the_output_cannot_be_created(tmp_path): + path = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + response = Mock(content=_gzipped_warc_record()) + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=response), \ + patch('cdx_toolkit.repackage.local_writer.aiofiles.open', side_effect=PermissionError('read-only')), \ + pytest.raises(PermissionError): + main(['repackage', path, '--prefix', str(tmp_path / 'out')]) + assert list(tmp_path.glob('out-*')) == [] + + +def test_repackage_with_wb_counts_failed_captures(tmp_path, capsys): + path = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE, IA_CAPTURE]) + fetched = [RuntimeError('404'), _make_fake_record()] + with patch.object(cdx_toolkit.cli.repackage, 'fetch_wb_warc', side_effect=fetched), \ + pytest.raises(SystemExit) as exc: + main(['repackage', path, '--wb', 'ia', '--prefix', str(tmp_path / 'out')]) + assert exc.value.code == 1 + assert '1 of 2 captures could not be fetched' in capsys.readouterr().err From a1833af0f18d125a7360232463e988ae7aa5c612 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:07:12 +0200 Subject: [PATCH 08/91] fix: validate range responses before writing them to the WARC An HTTP range response is only written if it holds exactly the requested bytes: status 206 with the requested Content-Range, or 200 for a range that is the whole file, and as many bytes as requested. A server that ignores the range, a redirect (not followed), a wrong range or a truncated body makes the capture fail. The HF reader uses the same check, the S3 reader checks the number of bytes. The tests serve WARC files with realistic range responses (tests/fake_warc_server.py). --- cdx_toolkit/repackage/data_classes.py | 9 +++- cdx_toolkit/repackage/hf_utils.py | 7 ++- cdx_toolkit/repackage/http_range.py | 27 +++++++++++ tests/fake_warc_server.py | 42 +++++++++++++++++ tests/repackage/test_data_classes.py | 22 +++++++++ tests/repackage/test_hf_utils.py | 5 ++- tests/repackage/test_warc_filter_pipeline.py | 47 +++++++++++++------- tests/unit/test_cli.py | 29 +++++++----- 8 files changed, 155 insertions(+), 33 deletions(-) create mode 100644 cdx_toolkit/repackage/http_range.py create mode 100644 tests/fake_warc_server.py diff --git a/cdx_toolkit/repackage/data_classes.py b/cdx_toolkit/repackage/data_classes.py index 879e85c..104977f 100644 --- a/cdx_toolkit/repackage/data_classes.py +++ b/cdx_toolkit/repackage/data_classes.py @@ -6,6 +6,7 @@ from typing import Optional, Tuple from cdx_toolkit.myrequests import myrequests_get +from cdx_toolkit.repackage.http_range import RangeError, check_range_response @dataclass @@ -105,7 +106,10 @@ async def ranged_get_bytes( max_attempts=max_attempts, base_backoff_seconds=base_backoff_seconds, ) - return await resp['Body'].read() + data = await resp['Body'].read() + if len(data) != length: + raise RangeError(f'{self.url}: requested {length} bytes from offset {offset}, got {len(data)}') + return data elif self.is_hf(): # read from a Hugging Face bucket (hf://) via the configured HF reader @@ -124,6 +128,9 @@ async def ranged_get_bytes( headers = {'Range': 'bytes={}-{}'.format(offset, end)} resp = myrequests_get(self.url, headers=headers) + check_range_response( + resp.status_code, resp.headers.get('Content-Range'), len(resp.content), offset, length, self.url + ) return resp.content diff --git a/cdx_toolkit/repackage/hf_utils.py b/cdx_toolkit/repackage/hf_utils.py index e197b4c..71517f7 100644 --- a/cdx_toolkit/repackage/hf_utils.py +++ b/cdx_toolkit/repackage/hf_utils.py @@ -23,6 +23,8 @@ from typing import Tuple from urllib.parse import parse_qs, urlparse +from cdx_toolkit.repackage.http_range import check_range_response + logger = logging.getLogger(__name__) HF_SCHEME = 'hf://' @@ -221,8 +223,9 @@ async def ranged_get( body = await resp.read() raise OSError(f'HF GET {resolve} -> {resp.status}: {body[:200]!r}') data = await resp.read() - if len(data) != length: - raise OSError(f'HF GET {resolve} returned {len(data)} bytes, expected {length}') + check_range_response( + resp.status, resp.headers.get('Content-Range'), len(data), offset, length, resolve + ) self.read_count += 1 return data except Exception: # noqa: BLE001 - retry any transient read error diff --git a/cdx_toolkit/repackage/http_range.py b/cdx_toolkit/repackage/http_range.py new file mode 100644 index 0000000..9017a2d --- /dev/null +++ b/cdx_toolkit/repackage/http_range.py @@ -0,0 +1,27 @@ +import re +from typing import Optional + + +class RangeError(OSError): + """A range request returned other bytes than requested.""" + + +def check_range_response(status: int, content_range: Optional[str], body_length: int, offset: int, length: int, + url: str) -> None: + """Raise RangeError unless an HTTP response holds exactly the bytes offset .. offset+length-1. + + A 206 response must name the requested range in Content-Range. A 200 response (the server ignored the + range) is only accepted for a range from offset 0 that is the whole file. Redirects are not followed: + a URL that redirects must be given as the redirect target.""" + end = offset + length - 1 + if status == 206: + match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+|\*)', (content_range or '').strip()) + if not match or (int(match.group(1)), int(match.group(2))) != (offset, end): + raise RangeError(f'{url}: requested bytes {offset}-{end}, got Content-Range {content_range!r}') + elif status == 200: + if offset != 0: + raise RangeError(f'{url}: the server ignored the range request for bytes {offset}-{end}') + else: + raise RangeError(f'{url}: unexpected status {status} for the range request for bytes {offset}-{end}') + if body_length != length: + raise RangeError(f'{url}: requested {length} bytes from offset {offset}, got {body_length}') diff --git a/tests/fake_warc_server.py b/tests/fake_warc_server.py new file mode 100644 index 0000000..49bce77 --- /dev/null +++ b/tests/fake_warc_server.py @@ -0,0 +1,42 @@ +"""WARC files served by fake HTTP range requests, with realistic responses (206, Content-Range, exact bytes).""" +from io import BytesIO +from typing import Dict, List +from unittest.mock import Mock + +from warcio import WARCWriter + +from cdx_toolkit.repackage.warc_utils import get_bytes_from_warc_record + + +def warc_record_bytes(url: str) -> bytes: + record = WARCWriter(None).create_warc_record(url, 'resource', payload=BytesIO(b'content of ' + url.encode())) + return get_bytes_from_warc_record(record, gzip=True) + + +class FakeWarcServer: + """WARC files (filename -> bytes) served under `prefix`; `get` stands in for myrequests_get.""" + + def __init__(self, prefix: str = 'https://data.example.org'): + self.prefix = prefix + self.files: Dict[str, bytes] = {} + self.requests: List[str] = [] + + def add_captures(self, filename: str, urls: List[str]) -> List[dict]: + """Append a WARC record per URL to the file, return the captures (CDX fields) of the records.""" + captures = [] + for url in urls: + data = warc_record_bytes(url) + offset = len(self.files.get(filename, b'')) + self.files[filename] = self.files.get(filename, b'') + data + captures.append({'url': url, 'filename': filename, 'offset': str(offset), 'length': str(len(data))}) + return captures + + def get(self, url, headers=None, **kwargs): + self.requests.append(url) + data = self.files.get(url[len(self.prefix):].lstrip('/')) + if data is None: + return Mock(status_code=404, headers={}, content=b'not found') + start, end = (int(n) for n in headers['Range'][len('bytes='):].split('-')) + body = data[start:end + 1] + content_range = f'bytes {start}-{start + len(body) - 1}/{len(data)}' + return Mock(status_code=206, headers={'Content-Range': content_range}, content=body) diff --git a/tests/repackage/test_data_classes.py b/tests/repackage/test_data_classes.py index 72854d7..7b36a8a 100644 --- a/tests/repackage/test_data_classes.py +++ b/tests/repackage/test_data_classes.py @@ -1,5 +1,6 @@ import pytest from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.http_range import RangeError, check_range_response def test_get_s3_bucket_and_key_from_http_job(): @@ -10,3 +11,24 @@ def test_get_s3_bucket_and_key_from_http_job(): ) with pytest.raises(ValueError): job.get_s3_bucket_and_key() + + +@pytest.mark.parametrize('status, content_range, body_length, offset, valid', [ + (206, 'bytes 10-19/100', 10, 10, True), + (206, 'bytes 10-19/*', 10, 10, True), + (200, None, 10, 0, True), # the whole file is the requested range + (200, None, 100, 10, False), # range ignored + (206, 'bytes 11-20/100', 10, 10, False), # wrong range + (206, 'bytes 10-19/100', 9, 10, False), # truncated body + (206, None, 10, 10, False), + (302, None, 0, 10, False), # redirects are not followed +]) +def test_check_range_response(status, content_range, body_length, offset, valid): + def check(): + check_range_response(status, content_range, body_length, offset, 10, 'https://example.org/x') + + if valid: + check() + else: + with pytest.raises(RangeError): + check() diff --git a/tests/repackage/test_hf_utils.py b/tests/repackage/test_hf_utils.py index 9b1de66..6b92b00 100644 --- a/tests/repackage/test_hf_utils.py +++ b/tests/repackage/test_hf_utils.py @@ -113,7 +113,8 @@ def get(self, url, headers=None): if self.get_statuses: return FakeResponse(self.get_statuses.pop(0)) start, end = map(int, headers['Range'][len('bytes='):].split('-')) - return FakeResponse(206, body=DATA[start:end + 1]) + body = DATA[start:end + 1] + return FakeResponse(206, {'Content-Range': f'bytes {start}-{start + len(body) - 1}/{len(DATA)}'}, body) def _read(reader, url, offset, length): @@ -213,7 +214,7 @@ def test_resolve_errors_are_retried_then_raised(): 'get_statuses, offset, length, error', [ ([500, 500, 500], 0, 4, 'HF GET .* -> 500'), - ([], 1000, 100, 'returned 24 bytes, expected 100'), # short read + ([], 1000, 100, 'requested bytes 1000-1099, got Content-Range'), # short read ], ) def test_read_errors_are_retried_then_raised(get_statuses, offset, length, error): diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index af3ea13..44c0a2d 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -5,12 +5,10 @@ import itertools import os import threading -from io import BytesIO from unittest.mock import Mock, patch import fsspec import pytest -from warcio import WARCWriter from warcio.archiveiterator import ArchiveIterator from cdx_toolkit.repackage import multiprocess @@ -18,22 +16,20 @@ from cdx_toolkit.repackage.local_writer import LocalFileWriter from cdx_toolkit.repackage.sources.records import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter -from cdx_toolkit.repackage.warc_utils import get_bytes_from_warc_record +from tests.fake_warc_server import FakeWarcServer -def warc_record_bytes(url): - record = WARCWriter(None).create_warc_record(url, 'resource', payload=BytesIO(b'content of ' + url.encode())) - return get_bytes_from_warc_record(record, gzip=True) +def make_server(): + """Four captures in two WARC files.""" + server = FakeWarcServer('https://data.example.org') + captures = [] + for i in range(4): + captures += server.add_captures(f'crawl-data/{i % 2}.warc.gz', [f'https://example.com/{i}']) + return server, captures -CAPTURES = [ - {'url': f'https://example.com/{i}', 'filename': f'crawl-data/{i % 2}.warc.gz', 'offset': str(i), 'length': '1'} - for i in range(4) -] - - -def fake_get(url, headers=None, **kwargs): - return Mock(content=warc_record_bytes(url + '#' + headers['Range'])) +SERVER, CAPTURES = make_server() +fake_get = SERVER.get def response_records(paths): @@ -60,6 +56,25 @@ def flaky_get(url, headers=None, **kwargs): assert len(response_records(glob.glob(str(tmp_path / 'out-*.warc.gz')))) == 3 +@pytest.mark.parametrize('response', [ + Mock(status_code=200, headers={}, content=b'x' * 100), # range ignored + Mock(status_code=301, headers={'Location': 'https://elsewhere.example.org/'}, content=b''), # redirect + Mock(status_code=206, headers={'Content-Range': 'bytes 0-0/100'}, content=b'x'), # truncated / wrong range + 'wrong-range', +]) +def test_invalid_range_responses_are_not_written(tmp_path, response): + capture = CAPTURES[1] # offset > 0 + offset, length = int(capture['offset']), int(capture['length']) + if response == 'wrong-range': + response = Mock(status_code=206, headers={'Content-Range': f'bytes {offset + 1}-{offset + length}/9999'}, + content=b'x' * length) + wf = WARCFilter(source=RecordsSource([capture], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}) + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=response): + assert wf.filter() == RepackageResult(requested=1, written=0, failed=1) + assert glob.glob(str(tmp_path / 'out-*')) == [] + + def test_writer_error_stops_the_pipeline_and_removes_the_file(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, max_file_size=1) @@ -137,7 +152,7 @@ def main(): def test_hf_reads_use_the_hf_reader(tmp_path): class FakeHfReader: async def ranged_get(self, url, offset, length, **kwargs): - return warc_record_bytes(f'{url}#{offset}') + return SERVER.files[url.split('/', 5)[-1]][offset:offset + length] @contextlib.asynccontextmanager async def fake_make_hf_reader(): @@ -149,7 +164,7 @@ async def fake_make_hf_reader(): with patch('cdx_toolkit.repackage.warc_filter.make_hf_reader', fake_make_hf_reader): assert wf.filter().written == 4 records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) - assert records[0].rec_headers.get_header('WARC-Target-URI').startswith('hf://buckets/commoncrawl/commoncrawl/') + assert sorted(r.rec_headers.get_header('WARC-Target-URI') for r in records) == sorted(c['url'] for c in CAPTURES) @pytest.mark.parametrize('uvloop_installed', [True, False]) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index dc2bf6c..fda29a1 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -30,6 +30,15 @@ def _make_fake_record(): ) +RECORD = cdx_toolkit.repackage.warc_utils.get_bytes_from_warc_record(_make_fake_record(), gzip=True) + + +def _range_response(offset=10): + """The response to a range request for RECORD at `offset` of a WARC file.""" + content_range = f'bytes {offset}-{offset + len(RECORD) - 1}/*' + return Mock(status_code=206, headers={'Content-Range': content_range}, content=RECORD) + + def _write_captures(path, captures): path.write_text(''.join(json.dumps(c) + '\n' for c in captures)) return str(path) @@ -38,7 +47,7 @@ def _write_captures(path, captures): IA_CAPTURE = {'url': 'http://example.com', 'timestamp': '20240101120000', 'status': '200', 'mime': 'text/html'} CC_CAPTURE = { 'url': 'http://example.com', 'timestamp': '20240101120000', 'status': '200', - 'filename': 'crawl-data/x.warc.gz', 'offset': '10', 'length': '20', + 'filename': 'crawl-data/x.warc.gz', 'offset': '10', 'length': str(len(RECORD)), } @@ -55,19 +64,15 @@ def test_repackage_with_wb_logs_revisit(tmp_path, caplog): assert list(tmp_path.glob('out-*.warc.gz')) -def _gzipped_warc_record(): - return cdx_toolkit.repackage.warc_utils.get_bytes_from_warc_record(_make_fake_record(), gzip=True) - - def test_repackage_fetches_by_warc_location(tmp_path): captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) - resp = Mock(content=_gzipped_warc_record()) + resp = _range_response() with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=resp) as get: main(['repackage', captures, '--prefix', str(tmp_path / 'out'), '--warc-download-prefix', 'https://x.org/']) assert get.call_args.args == ('https://x.org/crawl-data/x.warc.gz',) - assert get.call_args.kwargs == {'headers': {'Range': 'bytes=10-29'}} + assert get.call_args.kwargs == {'headers': {'Range': f'bytes=10-{9 + len(RECORD)}'}} with open(tmp_path / 'out-001.warc.gz', 'rb') as f: records = list(ArchiveIterator(f)) @@ -97,7 +102,7 @@ def test_repackage_default_prefix(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=Mock(content=_gzipped_warc_record())): + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=_range_response()): main(['repackage', captures]) assert [p.name for p in tmp_path.glob('*.warc.gz')] == ['REPACKAGE-001.warc.gz'] @@ -114,7 +119,7 @@ def test_index_to_parquet_and_repackage(tmp_path, capsys): main(['index', 'cc', '--url', 'example.com/*', '--output-format', 'parquet']) assert 'cannot be written to stdout' in capsys.readouterr().err - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=Mock(content=_gzipped_warc_record())): + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=_range_response()): main(['repackage', str(captures), '--prefix', str(tmp_path / 'out')]) with open(tmp_path / 'out-001.warc.gz', 'rb') as f: assert [r.rec_type for r in ArchiveIterator(f)] == ['warcinfo', 'response'] @@ -251,7 +256,7 @@ def test_verbose_before_and_after_subcommand(tmp_path): @pytest.mark.parametrize('n_failing', [1, 2]) def test_repackage_exits_1_if_captures_could_not_be_fetched(tmp_path, capsys, n_failing): - captures = [dict(CC_CAPTURE, offset=str(i)) for i in range(2)] + captures = [dict(CC_CAPTURE, offset=str(i * len(RECORD))) for i in range(2)] path = _write_captures(tmp_path / 'captures.jsonl', captures) calls = [] @@ -259,7 +264,7 @@ def get(url, headers=None, **kwargs): calls.append(url) if len(calls) <= n_failing: raise ConnectionError('network down') - return Mock(content=_gzipped_warc_record()) + return _range_response(offset=int(headers['Range'][len('bytes='):].split('-')[0])) with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=get), \ pytest.raises(SystemExit) as exc: @@ -271,7 +276,7 @@ def get(url, headers=None, **kwargs): def test_repackage_fails_if_the_output_cannot_be_created(tmp_path): path = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) - response = Mock(content=_gzipped_warc_record()) + response = _range_response() with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=response), \ patch('cdx_toolkit.repackage.local_writer.aiofiles.open', side_effect=PermissionError('read-only')), \ pytest.raises(PermissionError): From 2156bffd1004139a29e44e2e6005a45f3cb52e4e Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:09:14 +0200 Subject: [PATCH 09/91] fix: read full s3:// and hf:// filenames with the reader of their scheme The S3 client and the HF reader were only created if --warc-download-prefix was s3:// or hf://, so captures whose filename is a full s3:// or hf:// URL failed with the default https prefix. The readers are now created on first use, by the URL scheme of each range job, so mixed input works; the download prefix only resolves relative filenames and is no longer passed to the fetcher or the worker processes. --- cdx_toolkit/cli/repackage.py | 2 - cdx_toolkit/repackage/data_classes.py | 13 +- cdx_toolkit/repackage/multiprocess.py | 5 - cdx_toolkit/repackage/warc_filter.py | 132 +++++++++---------- tests/repackage/test_warc_filter_pipeline.py | 80 +++++++++-- tests/unit/test_aws.py | 8 +- tests/unit/test_cli.py | 2 +- tests/unit/test_usage_errors.py | 8 +- 8 files changed, 142 insertions(+), 108 deletions(-) diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index 1efd5d9..27a2e11 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -196,7 +196,6 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): max_attempts=5, uvloop=os.environ.get('CDXT_UVLOOP') == '1', max_file_size=args.size, - warc_download_prefix=args.warc_download_prefix, ) from cdx_toolkit.repackage.warc_filter import WARCFilter @@ -208,7 +207,6 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): writer_subprefix=args.subprefix, write_paths_as_metadata_records=args.add_metadata_record, log_every_n=args.log_every_n, - warc_download_prefix=args.warc_download_prefix, n_parallel_readers=args.parallel_readers, max_file_size=args.size, ).filter() diff --git a/cdx_toolkit/repackage/data_classes.py b/cdx_toolkit/repackage/data_classes.py index 104977f..9e66f4f 100644 --- a/cdx_toolkit/repackage/data_classes.py +++ b/cdx_toolkit/repackage/data_classes.py @@ -88,10 +88,9 @@ async def ranged_get_bytes( self, max_attempts: int, base_backoff_seconds: float, - s3_client=None, - hf_reader=None, + readers=None, ) -> bytes: - """Ranged get request to S3 (aioboto3), an HF bucket, or HTTP.""" + """Ranged get request to S3 (aioboto3), an HF bucket, or HTTP; `readers` provides the S3 and HF readers.""" offset = self.offset length = self.length @@ -100,6 +99,9 @@ async def ranged_get_bytes( if self.is_s3(): # read from S3 bucket, key = self.get_s3_bucket_and_key() + if readers is None: + raise ValueError('s3:// range read requires readers (none were provided)') + s3_client = await readers.s3() resp = await with_retries( lambda: s3_client.get_object(Bucket=bucket, Key=key, Range=f'bytes={offset}-{end}'), op_name=f'ranged_get {bucket}/{key}[{offset}:{end}]', @@ -113,8 +115,9 @@ async def ranged_get_bytes( elif self.is_hf(): # read from a Hugging Face bucket (hf://) via the configured HF reader - if hf_reader is None: - raise ValueError('hf:// range read requires an hf_reader (none was provided)') + if readers is None: + raise ValueError('hf:// range read requires readers (none were provided)') + hf_reader = await readers.hf() return await hf_reader.ranged_get( self.url, offset, diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index 07690b0..51076bf 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -85,9 +85,6 @@ def _run_worker(cfg: Dict) -> RepackageResult: writer_subprefix=cfg['subprefix'], write_paths_as_metadata_records=cfg['write_paths_as_metadata_records'], log_every_n=cfg['log_every_n'], - # Shard CSVs hold full URLs (warc_filename), so CsvSource ignores this; it is - # still needed so WARCFilter knows the read scheme (S3 vs hf://). - warc_download_prefix=cfg['warc_download_prefix'], n_parallel_readers=cfg['readers'], aws_region_name=cfg['aws_region_name'], max_attempts=cfg['max_attempts'], @@ -133,7 +130,6 @@ def run_multiprocess_repackage( max_attempts: int, uvloop: bool, max_file_size: Optional[int] = 1_000_000_000, - warc_download_prefix: Optional[str] = None, ) -> RepackageResult: """Shard -> N worker processes (rotating at --size) writing one global [-]-NNN.warc.gz series.""" @@ -165,7 +161,6 @@ def run_multiprocess_repackage( max_attempts=max_attempts, max_file_size=max_file_size, uvloop=uvloop, - warc_download_prefix=warc_download_prefix, ) for shard_csv in shard_csvs ] diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py index 707511b..d7ce322 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/warc_filter.py @@ -12,7 +12,7 @@ from cdx_toolkit.repackage.s3_utils import ( is_s3_url, ) -from cdx_toolkit.repackage.hf_utils import is_hf_url, make_hf_reader +from cdx_toolkit.repackage.hf_utils import make_hf_reader from cdx_toolkit.repackage.data_classes import RangeJob, RangePayload, RepackageResult, ThroughputTracker from cdx_toolkit.repackage.warc_utils import create_new_writer_with_header from cdx_toolkit.repackage.sources.base import RangeJobSource @@ -24,6 +24,33 @@ logger = logging.getLogger(__name__) +class Readers: + """The readers of WARC ranges, by the URL scheme of a range job, each created when it is first needed: + an S3 client for s3:// and the HF reader for hf:// (http(s):// needs none). They are closed with `stack`.""" + + def __init__(self, stack: contextlib.AsyncExitStack, s3_config_factory: Callable): + self.stack = stack + self.s3_config_factory = s3_config_factory + self._s3 = None + self._hf = None + self._lock = asyncio.Lock() + + async def s3(self): + async with self._lock: + if self._s3 is None: + import aioboto3 + + client = aioboto3.Session().client('s3', config=self.s3_config_factory()) + self._s3 = await self.stack.enter_async_context(client) + return self._s3 + + async def hf(self): + async with self._lock: + if self._hf is None: + self._hf = await self.stack.enter_async_context(make_hf_reader()) + return self._hf + + class WARCFilter: """Filter or extract specific records from WARC files based on CDX indexes. @@ -49,7 +76,6 @@ def __init__( writer_subprefix: Optional[str] = None, write_paths_as_metadata_records: Optional[List[str]] = None, log_every_n: int = 1000, - warc_download_prefix: Optional[str] = None, n_parallel_readers: int = 1, max_attempts: int = 5, base_backoff_seconds: float = 0.5, @@ -72,7 +98,6 @@ def __init__( writer_subprefix: Optional subprefix for writer output paths. write_paths_as_metadata_records: Optional list of file paths to write as metadata records. log_every_n: Log progress every N records. - warc_download_prefix: Optional prefix to prepend to WARC URLs. n_parallel_readers: Number of async readers per process (one writer per process). max_attempts: Maximum retry attempts for failed operations. base_backoff_seconds: Base backoff time in seconds for retries. @@ -93,7 +118,6 @@ def __init__( self.writer_subprefix = writer_subprefix self.write_paths_as_metadata_records = write_paths_as_metadata_records self.log_every_n = log_every_n - self.warc_download_prefix = warc_download_prefix # self.writer_kwargs = writer_kwargs self.range_jobs_queue_size = range_jobs_queue_size @@ -135,76 +159,44 @@ def filter(self) -> RepackageResult: logger.warning('CDXT_UVLOOP=1 but uvloop is not installed; using default asyncio loop') return runner(self.filter_async()) - def needs_aws(self) -> bool: - """Returns true if the read/write (stage 2/3) S3 clients are needed. - - Sources own their own stage-1 resource (Athena client / DuckDB connection / - fsspec), so this only concerns WARC reads and output writes. - """ - return is_s3_url(self.warc_download_prefix) or is_s3_url(self.prefix_path) - - async def get_aws_clients(self) -> Optional[Dict]: - """Return async S3 clients for WARC reads/writes if needed. - - Stage-1 clients/connections are owned by the source, so this only builds the - read/write S3 clients used to fetch WARC ranges and write output. - """ - if self.needs_aws(): - import aioboto3 - - session = aioboto3.Session() - - # region, retries and S3 settings: Common Crawl's recommended defaults, unless the - # user configured them (see cdx_toolkit.aws); an explicit aws_region_name wins - # High-throughput config for range reads - read_config = botocore_config( - region_name=self.aws_region_name, - max_pool_connections=self.num_readers * 3, - read_timeout=300, - tcp_keepalive=True, - ) - - # Optimized config for multipart uploads (single writer) - write_config = botocore_config( - region_name=self.aws_region_name, - max_pool_connections=8, - read_timeout=120, - connect_timeout=10, - ) + def s3_read_config(self): + """botocore config of the S3 client for range reads: high throughput.""" + # region, retries and S3 settings: Common Crawl's recommended defaults, unless the + # user configured them (see cdx_toolkit.aws); an explicit aws_region_name wins + return botocore_config( + region_name=self.aws_region_name, + max_pool_connections=self.num_readers * 3, + read_timeout=300, + tcp_keepalive=True, + ) - return { - 'read': session.client('s3', config=read_config), - 'write': session.client('s3', config=write_config), - } - else: - return None + def s3_write_config(self): + """botocore config of the S3 client for multipart uploads (single writer).""" + return botocore_config( + region_name=self.aws_region_name, + max_pool_connections=8, + read_timeout=120, + connect_timeout=10, + ) async def filter_async(self) -> RepackageResult: """Filter process using a three stage approach (job generator, warc reader, warc writer).""" range_jobs_queue: asyncio.Queue = asyncio.Queue(maxsize=self.range_jobs_queue_size) warc_records_queue: asyncio.Queue = asyncio.Queue(maxsize=self.warc_records_queue_size) - read_is_hf = is_hf_url(self.warc_download_prefix) - async with contextlib.AsyncExitStack() as stack: - read_s3_client = write_s3_client = None - # S3 read and/or write clients (built together when either side is S3). - if self.needs_aws(): - clients = await self.get_aws_clients() - read_s3_client = await stack.enter_async_context(clients['read']) - write_s3_client = await stack.enter_async_context(clients['write']) - - # HF reader for hf:// downloads (CDN ranged GETs, one resolve per WARC file). - hf_reader = None - if read_is_hf: - hf_reader = await stack.enter_async_context(make_hf_reader()) + write_s3_client = None + if is_s3_url(self.prefix_path): + import aioboto3 + + client = aioboto3.Session().client('s3', config=self.s3_write_config()) + write_s3_client = await stack.enter_async_context(client) return await self._run_filter_pipeline( range_jobs_queue=range_jobs_queue, warc_records_queue=warc_records_queue, - read_s3_client=read_s3_client, + readers=Readers(stack, self.s3_read_config), write_s3_client=write_s3_client, - hf_reader=hf_reader, ) async def _produce_range_jobs(self, range_jobs_queue: asyncio.Queue, stop: threading.Event) -> int: @@ -242,9 +234,8 @@ async def _run_filter_pipeline( self, range_jobs_queue: asyncio.Queue, warc_records_queue: asyncio.Queue, - read_s3_client=None, + readers: Readers, write_s3_client=None, - hf_reader=None, ) -> RepackageResult: """Run the actual filter pipeline with or without S3 client. @@ -254,9 +245,8 @@ async def _run_filter_pipeline( Args: range_jobs_queue: Queue for range jobs from the source. warc_records_queue: Queue for WARC record payloads. - read_s3_client: Optional S3 client for reads from S3. + readers: The readers of WARC ranges (by URL scheme). write_s3_client: Optional S3 client for writes S3. - hf_reader: Optional HF reader for hf:// downloads. """ logger.info('Starting job generator, %d WARC readers, 1 WARC writer', self.num_readers) stop = threading.Event() @@ -271,8 +261,7 @@ async def _run_filter_pipeline( reader_id=i, range_jobs_queue=range_jobs_queue, warc_records_queue=warc_records_queue, - s3_client=read_s3_client, - hf_reader=hf_reader, + readers=readers, ) ) for i in range(self.num_readers) @@ -338,8 +327,7 @@ async def read_warc_records( reader_id: int, range_jobs_queue: asyncio.Queue, warc_records_queue: asyncio.Queue, - s3_client=None, - hf_reader=None, + readers: Readers, ) -> dict: """Read WARC records based on range jobs -> enqueue RangePayload. @@ -347,8 +335,7 @@ async def read_warc_records( reader_id: Unique identifier for this reader task. range_jobs_queue: Queue to read RangeJob objects from. warc_records_queue: Queue to put RangePayload objects into. - s3_client: Optional S3 client for reading WARC files from S3. - hf_reader: Optional HF reader for reading WARC ranges from hf:// buckets. + readers: The readers of WARC ranges (by URL scheme). Returns: dict: Statistics dictionary with reader_id and throughput stats. @@ -377,8 +364,7 @@ async def read_warc_records( data = await job.ranged_get_bytes( self.max_attempts, self.base_backoff_seconds, - s3_client=s3_client, - hf_reader=hf_reader, + readers=readers, ) tracker.add(bytes_count=len(data), records_count=job.records_count) counter += 1 diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 44c0a2d..3c7f79f 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -150,17 +150,8 @@ def main(): def test_hf_reads_use_the_hf_reader(tmp_path): - class FakeHfReader: - async def ranged_get(self, url, offset, length, **kwargs): - return SERVER.files[url.split('/', 5)[-1]][offset:offset + length] - - @contextlib.asynccontextmanager - async def fake_make_hf_reader(): - yield FakeHfReader() - wf = WARCFilter(source=RecordsSource(CAPTURES, 'hf://buckets/commoncrawl/commoncrawl'), - prefix_path=str(tmp_path / 'out'), writer_info={}, - warc_download_prefix='hf://buckets/commoncrawl/commoncrawl') + prefix_path=str(tmp_path / 'out'), writer_info={}) with patch('cdx_toolkit.repackage.warc_filter.make_hf_reader', fake_make_hf_reader): assert wf.filter().written == 4 records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) @@ -179,10 +170,71 @@ def test_uvloop(monkeypatch, caplog, uvloop_installed): assert ('Using uvloop event loop' if uvloop_installed else 'uvloop is not installed') in caplog.text -def test_no_aws_clients_without_s3(): - wf = WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, - warc_download_prefix='https://data.commoncrawl.org') - assert asyncio.run(wf.get_aws_clients()) is None +class FakeS3Client: + """aioboto3 S3 client stand-in that serves the files of SERVER.""" + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get_object(self, Bucket, Key, Range): + start, end = (int(n) for n in Range[len('bytes='):].split('-')) + + class Body: + async def read(self): + return SERVER.files[Key][start:end + 1] + + return {'Body': Body()} + + +class FakeHfReader: + async def ranged_get(self, url, offset, length, **kwargs): + return SERVER.files[url.split('/', 5)[-1]][offset:offset + length] + + +@contextlib.asynccontextmanager +async def fake_make_hf_reader(): + yield FakeHfReader() + + +def mixed_scheme_captures(): + """The captures with full s3:// and hf:// filenames, and a relative one (default download prefix).""" + s3, hf, relative = CAPTURES[0], CAPTURES[1], CAPTURES[2] + return [dict(s3, filename='s3://commoncrawl/' + s3['filename']), + dict(hf, filename='hf://buckets/commoncrawl/commoncrawl/' + hf['filename']), relative] + + +@pytest.mark.parametrize('processes', [1, 2]) +def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): + pytest.importorskip('aioboto3') + source = RecordsSource(mixed_scheme_captures(), 'https://data.example.org') + with patch('aioboto3.Session') as session, \ + patch('cdx_toolkit.repackage.warc_filter.make_hf_reader', fake_make_hf_reader), \ + patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get), \ + patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor): + session.return_value.client.return_value = FakeS3Client() + if processes == 1: + result = WARCFilter(source=source, prefix_path=str(tmp_path / 'out'), writer_info={}).filter() + else: + result = multiprocess.run_multiprocess_repackage( + source=source, n_processes=processes, readers_per_process=1, prefix_path=str(tmp_path / 'out'), + writer_info={}, writer_subprefix=None, write_paths_as_metadata_records=None, log_every_n=0, + max_attempts=1, uvloop=False, + ) + assert result == RepackageResult(requested=3, written=3) + records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) + assert sorted(r.rec_headers.get_header('WARC-Target-URI') for r in records) == \ + sorted(c['url'] for c in CAPTURES[:3]) + + +def test_no_s3_client_without_s3(tmp_path): + with patch('aioboto3.Session') as session, \ + patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}).filter() + session.assert_not_called() class InlineExecutor: diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py index 99ceaea..59d1fb7 100644 --- a/tests/unit/test_aws.py +++ b/tests/unit/test_aws.py @@ -1,5 +1,4 @@ """Common Crawl's S3 defaults (cdx_toolkit.aws): used for the settings the user did not configure.""" -import asyncio from unittest.mock import patch import pytest @@ -133,11 +132,8 @@ def test_warc_fetcher_s3_clients(no_user_config): from cdx_toolkit.repackage.sources.records import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter - wf = WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, - warc_download_prefix='s3://commoncrawl', n_parallel_readers=4) - with patch('aioboto3.Session') as session: - asyncio.run(wf.get_aws_clients()) - read_config, write_config = (c.kwargs['config'] for c in session.return_value.client.call_args_list) + wf = WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=4) + read_config, write_config = wf.s3_read_config(), wf.s3_write_config() for config in (read_config, write_config): assert config.region_name == 'us-east-1' assert config.retries == {'mode': 'adaptive', 'total_max_attempts': 100000} diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index fda29a1..af56e85 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -142,7 +142,7 @@ def test_repackage_options_are_passed_to_the_warc_fetcher(tmp_path): assert kwargs['max_file_size'] == 1024 * 1024 assert kwargs['write_paths_as_metadata_records'] == ['a.txt'] assert kwargs['n_parallel_readers'] == 4 - assert kwargs['warc_download_prefix'] == 's3://commoncrawl' + assert [job.url for job in kwargs['source'].iter_range_jobs()] == [] info = kwargs['writer_info'] assert info['isPartOf'] == 'FOO-BAR' assert info['creator'] == 'me' diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py index 6fe8995..291a07f 100644 --- a/tests/unit/test_usage_errors.py +++ b/tests/unit/test_usage_errors.py @@ -124,11 +124,15 @@ def test_parquet_batches_exactly_full(tmp_path, monkeypatch): def test_hf_range_job_needs_a_reader(): job = RangeJob(url='hf://buckets/ns/name/x.warc.gz', offset=0, length=1) - with pytest.raises(ValueError, match='requires an hf_reader'): + with pytest.raises(ValueError, match='requires readers'): asyncio.run(job.ranged_get_bytes(1, 0)) class Reader: async def ranged_get(self, url, offset, length, **kwargs): return b'x' - assert asyncio.run(job.ranged_get_bytes(1, 0, hf_reader=Reader())) == b'x' + class Readers: + async def hf(self): + return Reader() + + assert asyncio.run(job.ranged_get_bytes(1, 0, readers=Readers())) == b'x' From df43f6bcdbd6bd48284a5ce9f3dbc934f94a2736 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:10:22 +0200 Subject: [PATCH 10/91] fix: SQL results without a url column work unless --url-fgrep is used The URL filters read record['url'] even when neither --url-fgrep nor --url-fgrepv was given, so a --sql query that selects only the WARC location crashed with a KeyError. Records now pass through untouched without URL filters; with a URL filter, a record without url is a usage error that names the missing column. --- cdx_toolkit/indexes/base.py | 7 ++++++- tests/unit/test_sql_indexes.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py index 85a05f8..83b1cc1 100644 --- a/cdx_toolkit/indexes/base.py +++ b/cdx_toolkit/indexes/base.py @@ -48,8 +48,13 @@ def iter_index_records(backend: IndexBackend, query: IndexQuery) -> Iterator[Map def filter_by_url(records: Iterable[Mapping], url_fgrep=None, url_fgrepv=None) -> Iterator[Mapping]: + if not (url_fgrep or url_fgrepv): + yield from records # records need no url, e.g. of a --sql query that selects only the WARC location + return for record in records: - url = record['url'] + url = record.get('url') + if url is None: + raise UsageError('--url-fgrep and --url-fgrepv need captures with a url (e.g. SELECT url in --sql)') if url_fgrep and url_fgrep not in url: LOGGER.debug('skipping due to --url-fgrep: %s', url) continue diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index 0908262..25a856a 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -303,3 +303,32 @@ def urls(*args): assert urls('--url', 'example.com/x') == ['https://example.com/x'] assert urls('--url', 'example.com/*') == ['https://example.com/', 'https://example.com/', 'https://example.com/x'] assert urls('--url', '*.example.org', '--url', 'example.com/x') == ['https://example.com/x', 'https://example.org/'] + + +@requires_duckdb +def test_sql_without_url_column_to_repackage(parquet_index, tmp_path, capsys): + """A --sql query that selects only the WARC location is enough for repackaging.""" + from tests.fake_warc_server import FakeWarcServer + + server = FakeWarcServer('https://data.example.org') + capture, = server.add_captures('a.warc.gz', ['https://example.com/x']) + files = f'{parquet_index}/*/*/*.parquet' + sql = tmp_path / 'query.sql' + sql.write_text( + f"SELECT warc_filename, {capture['offset']} AS warc_record_offset, {capture['length']} AS warc_record_length " + f"FROM read_parquet('{files}') WHERE url = 'https://example.com/x' AND crawl = 'CC-MAIN-2024-33'" + ) + captures = tmp_path / 'captures.jsonl' + main(['index', files, '--sql', str(sql), '--yes', '-o', str(captures)]) + assert [json.loads(line) for line in captures.read_text().splitlines()] == [ + {'filename': 'a.warc.gz', 'offset': capture['offset'], 'length': capture['length']} + ] + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=server.get): + main(['repackage', str(captures), '--prefix', str(tmp_path / 'out'), + '--warc-download-prefix', 'https://data.example.org']) + assert len(list(tmp_path.glob('out-*.warc.gz'))) == 1 + + with pytest.raises(SystemExit) as exc: + main(['index', files, '--sql', str(sql), '--yes', '--url-fgrep', 'example']) + assert exc.value.code == 2 + assert '--url-fgrep and --url-fgrepv need captures with a url' in capsys.readouterr().err From fc5bfc752508ab27fac00d7079a194d62be88f60 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:11:51 +0200 Subject: [PATCH 11/91] fix: CSV, TSV and Parquet output keep fields that only later captures have The columns were the fields of the first capture, and other fields of later captures were silently dropped. Now: - SQL results have a fixed schema: NULL columns stay in the record as None (empty in CSV/TSV, left out in JSONL/CDXJ as before), so the first record has all columns - for the CDX API and CDX files, the captures are first written to a temporary file to find all fields - an explicit --fields still selects the columns - the CSV writer raises instead of dropping an unexpected field --- cdx_toolkit/cli/index.py | 7 ++++-- cdx_toolkit/indexes/base.py | 2 ++ cdx_toolkit/indexes/sql.py | 9 ++++--- cdx_toolkit/record_io.py | 44 +++++++++++++++++++++++++++++----- tests/unit/test_record_io.py | 44 ++++++++++++++++++++++++++++++++++ tests/unit/test_sql_indexes.py | 3 ++- 6 files changed, 97 insertions(+), 12 deletions(-) diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index b11186f..282941e 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -127,7 +127,8 @@ def add_parser(subparsers): '--output-format', choices=FORMAT_CHOICES, help=('output format (default: detected from the --output extension, jsonl for stdout); cdx is the same as ' "cdxj: Common Crawl's index files have the .cdx extension but use the CDXJ format (crawl index JSON); " - 'parquet needs cdx_toolkit[parquet]'), + 'parquet needs cdx_toolkit[parquet]; without --fields, CSV, TSV and Parquet output of the CDX API and ' + 'CDX files is written once all captures are read (to include all fields)'), ) output.add_argument( '--fields', type=parse_fields, metavar='FIELD,...', @@ -247,5 +248,7 @@ def run(args: Namespace, cmdline: str): fh.write(f'{count}\n') return - n = write_records(iter_index_records(backend, query), args.output, output_format, args.fields) + n = write_records( + iter_index_records(backend, query), args.output, output_format, args.fields, fixed_schema=backend.fixed_schema + ) LOGGER.info('wrote %d captures to %s', n, args.output) diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py index 83b1cc1..8e5cbcf 100644 --- a/cdx_toolkit/indexes/base.py +++ b/cdx_toolkit/indexes/base.py @@ -17,6 +17,8 @@ class IndexBackend(ABC): name: str = '' supported_options: FrozenSet[str] = frozenset() + # True if all records have the same fields (in the same order), e.g. the columns of a SQL result + fixed_schema: bool = False def validate(self, query: IndexQuery) -> None: unsupported = sorted(query.options_used() - self.supported_options) diff --git a/cdx_toolkit/indexes/sql.py b/cdx_toolkit/indexes/sql.py index b066ccb..f7f2141 100644 --- a/cdx_toolkit/indexes/sql.py +++ b/cdx_toolkit/indexes/sql.py @@ -82,12 +82,14 @@ def confirm_cost(estimate: Optional[CostEstimate], confirmed: bool) -> None: def normalize_row(row: Mapping) -> dict: - """Columnar index row -> record with CDX field names; timestamps as YYYYMMDDhhmmss strings.""" + """Columnar index row -> record with CDX field names; timestamps as YYYYMMDDhhmmss strings. + + NULL values stay None, so that every record has all columns (see IndexBackend.fixed_schema).""" record = {} for column, value in row.items(): if value is None: - continue - if isinstance(value, datetime.datetime): + pass + elif isinstance(value, datetime.datetime): if value.tzinfo is not None: value = value.astimezone(datetime.timezone.utc) value = value.strftime('%Y%m%d%H%M%S') @@ -150,6 +152,7 @@ class SqlBackend(IndexBackend): """Common logic of the SQL backends: guided query building, cost guard, row normalization.""" engine = '' + fixed_schema = True supported_options = frozenset({ 'urls', 'hostnames', 'domains', 'crawl', 'from_ts', 'to', 'languages', 'limit', 'sql', 'url_fgrep', 'url_fgrepv', diff --git a/cdx_toolkit/record_io.py b/cdx_toolkit/record_io.py index 5767947..05cb442 100644 --- a/cdx_toolkit/record_io.py +++ b/cdx_toolkit/record_io.py @@ -10,7 +10,8 @@ import json import re import sys -from typing import IO, Iterable, Iterator, List, Mapping, Optional +import tempfile +from typing import IO, Iterable, Iterator, List, Mapping, Optional, Tuple import fsspec @@ -72,7 +73,11 @@ def open_text(path: str, mode: str) -> IO: class RecordWriter: - """Write records in one of TEXT_FORMATS; `fields` selects and orders the fields (default: all).""" + """Write records in one of TEXT_FORMATS; `fields` selects and orders the fields (default: all). + + Fields with the value None are left out (JSONL, CDXJ) or empty (CSV, TSV). Without `fields`, the CSV/TSV + columns are the fields of the first record, and a later record with another field is an error (see + write_records, which finds all fields first).""" def __init__(self, fh: IO, fmt: str, fields: Optional[List[str]] = None): fmt = canonical_format(fmt) @@ -86,6 +91,8 @@ def __init__(self, fh: IO, fmt: str, fields: Optional[List[str]] = None): def write(self, record: Mapping) -> None: record = dict(record) selected = {k: record[k] for k in self.fields if k in record} if self.fields else record + if self.fmt in ('jsonl', 'cdxj'): + selected = {k: v for k, v in selected.items() if v is not None} if self.fmt == 'jsonl': self.fh.write(json.dumps(selected, sort_keys=True, default=str) + '\n') @@ -102,7 +109,7 @@ def write(self, record: Mapping) -> None: fieldnames=self.fields or list(record.keys()), delimiter='\t' if self.fmt == 'tsv' else ',', lineterminator='\n', - extrasaction='ignore', + extrasaction='ignore' if self.fields else 'raise', ) self._csv_writer.writeheader() self._csv_writer.writerow(selected) @@ -208,7 +215,7 @@ def import_pyarrow(): def write_parquet(records: Iterable[Mapping], path: str, fields: Optional[List[str]] = None) -> int: """Write records to a Parquet file (all columns are strings), returns the number of records written. - Without `fields`, the columns are the fields of the first batch of records.""" + Without `fields`, the columns are the fields of the first batch of records (see write_records).""" if path == '-': raise ValueError('Parquet output cannot be written to stdout, please set --output') pa = import_pyarrow() @@ -251,9 +258,34 @@ def read_parquet(path: str) -> Iterator[dict]: yield normalize_field_names({k: v for k, v in record.items() if v is not None}) -def write_records(records: Iterable[Mapping], path: str, fmt: str, fields: Optional[List[str]] = None) -> int: - """Write records to a file or stdout ('-'), returns the number of records written.""" +def spool_records(records: Iterable[Mapping]) -> Tuple[List[str], Iterator[dict]]: + """Write the records to a temporary file; return all their fields (in the order seen) and the records.""" + spool = tempfile.TemporaryFile('w+', encoding='utf-8') + fields = {} + for record in records: + record = dict(record) # e.g. a CaptureObject of the CDX API + fields.update(dict.fromkeys(record)) + spool.write(json.dumps(record, default=str) + '\n') + spool.seek(0) + + def read_back(): + with spool: + for line in spool: + yield json.loads(line) + + return list(fields), read_back() + + +def write_records(records: Iterable[Mapping], path: str, fmt: str, fields: Optional[List[str]] = None, + fixed_schema: bool = False) -> int: + """Write records to a file or stdout ('-'), returns the number of records written. + + The columns of CSV, TSV and Parquet are `fields`; without them, the fields of the first record if all + records have the same fields (`fixed_schema`, e.g. SQL results), otherwise all fields of all records, + found by reading the records into a temporary file first.""" fmt = canonical_format(fmt) + if fmt in ('csv', 'tsv', 'parquet') and not fields and not fixed_schema: + fields, records = spool_records(records) if fmt == 'parquet': return write_parquet(records, path, fields) n = 0 diff --git a/tests/unit/test_record_io.py b/tests/unit/test_record_io.py index a88ebe6..53292f7 100644 --- a/tests/unit/test_record_io.py +++ b/tests/unit/test_record_io.py @@ -144,3 +144,47 @@ def test_is_glob(path, glob): def test_no_globs_over_http(): with pytest.raises(ValueError, match='globs do not work with http'): list(read_records('https://data.commoncrawl.org/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz')) + + +@pytest.mark.parametrize('fmt', ['csv', 'tsv', 'parquet']) +def test_fields_that_appear_later_are_kept(tmp_path, fmt): + if fmt == 'parquet': + pytest.importorskip('pyarrow') + records = [{'url': 'http://example.com/'}, {'url': 'http://example.com/x', 'redirect': 'http://example.com/y'}] + path = str(tmp_path / f'out.{fmt}') + assert write_records(iter(records), path, fmt) == 2 + assert list(read_records(path)) == [dict(records[0], **({} if fmt == 'parquet' else {'redirect': ''})), records[1]] + + +@pytest.mark.parametrize('fmt', ['csv', 'tsv']) +def test_fixed_schema_with_null_values(tmp_path, fmt): + """SQL results: every record has all columns, NULL is None (empty in CSV/TSV, left out in JSONL).""" + records = [{'url': 'http://example.com/', 'redirect': None}, {'url': 'http://example.com/x', 'redirect': 'y'}] + path = str(tmp_path / f'out.{fmt}') + write_records(iter(records), path, fmt, fixed_schema=True) + assert list(read_records(path)) == [{'url': 'http://example.com/', 'redirect': ''}, records[1]] + + path = str(tmp_path / 'out.jsonl') + write_records(iter(records), path, 'jsonl', fixed_schema=True) + assert list(read_records(path)) == [{'url': 'http://example.com/'}, records[1]] + + +def test_csv_writer_does_not_drop_fields(): + writer = RecordWriter(io.StringIO(), 'csv') + writer.write({'url': 'a'}) + with pytest.raises(ValueError): + writer.write({'url': 'b', 'redirect': 'c'}) + + +def test_csv_of_capture_objects(tmp_path): + """CDX API captures are CaptureObjects (mappings, not dicts).""" + from cdx_toolkit import CaptureObject + + captures = [CaptureObject({'url': 'http://example.com/', 'status': '200'}, wb=None), + CaptureObject({'url': 'http://example.com/x', 'status': '200', 'redirect': 'y'}, wb=None)] + path = str(tmp_path / 'out.csv') + write_records(iter(captures), path, 'csv') + assert list(read_records(path)) == [ + {'url': 'http://example.com/', 'status': '200', 'redirect': ''}, + {'url': 'http://example.com/x', 'status': '200', 'redirect': 'y'}, + ] diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index 25a856a..c548d55 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -67,7 +67,8 @@ def test_normalize_row(): 'my_column': 'x', } assert normalize_row(row) == { - 'url': 'https://example.com/', 'timestamp': '20240713102233', 'status': '200', 'offset': '10', 'my_column': 'x', + 'url': 'https://example.com/', 'timestamp': '20240713102233', 'status': '200', 'offset': '10', + 'languages': None, 'my_column': 'x', } # Athena returns strings assert normalize_row({'fetch_time': '2024-07-13 10:22:33.000'}) == {'timestamp': '20240713102233'} From 05a90e4d9484b4ff9df7901d7cb30cf7c54d6b54 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:14:02 +0200 Subject: [PATCH 12/91] fix: SQL --domain matches a subdomain selector and its subdomains --domain blog.example.com matched url_host_registered_domain = 'blog.example.com', which no capture has (their registered domain is example.com), while CDX files selected the captures below blog.example.com. The SQL query now matches the same SURT prefixes as CDX files (with a host boundary), and keeps a registered-domain condition (one of the domain's suffixes) so the engines can still prune. A shared test runs the same selectors against CDX files and DuckDB. --- cdx_toolkit/repackage/sources/sql_base.py | 26 +++++-- tests/repackage/test_athena_query_builder.py | 4 +- tests/unit/test_selectors.py | 71 ++++++++++++++++++++ tests/unit/test_sql_indexes.py | 15 +++-- 4 files changed, 104 insertions(+), 12 deletions(-) create mode 100644 tests/unit/test_selectors.py diff --git a/cdx_toolkit/repackage/sources/sql_base.py b/cdx_toolkit/repackage/sources/sql_base.py index 5724de0..7e1cb88 100644 --- a/cdx_toolkit/repackage/sources/sql_base.py +++ b/cdx_toolkit/repackage/sources/sql_base.py @@ -49,6 +49,24 @@ def surt_tld(surt: str) -> str: return surt.split(',', 1)[0].split(')', 1)[0] +def domain_predicate(domain: str) -> str: + """Captures of the domain and its subdomains (like --domain for CDX files), e.g. for blog.example.com: + + url_host_registered_domain IN ('blog.example.com', 'example.com') + AND (url_surtkey LIKE 'com,example,blog)%' OR url_surtkey LIKE 'com,example,blog,%') + + The SURT prefixes give the exact host boundary; the registered domain, which is one of the suffixes of the + domain, makes the query cheaper (an equality the engines can push down).""" + from cdx_toolkit.indexes.patterns import domain_to_surts + + surts = domain_to_surts(domain) + labels = surts[0][:-1].split(',')[::-1] # canonical, e.g. without www. + suffixes = ['.'.join(labels[i:]) for i in range(len(labels) - 1)] or labels + registered = ', '.join(escape_sql_literal(suffix) for suffix in suffixes) + surt_match = ' OR '.join(f'url_surtkey {sql_like_prefix(surt)}' for surt in surts) + return f'(url_host_registered_domain IN ({registered}) AND ({surt_match}))' + + def build_where_sql( url_host_names: Optional[List[str]] = None, crawls: Optional[List[str]] = None, @@ -60,9 +78,9 @@ def build_where_sql( """Build the WHERE body (without the `WHERE` keyword) shared by all SQL engines. The guided filter matches on `url_host_name` (exact host, e.g. www.example.com), - `url_host_registered_domain` (e.g. example.com, which also covers its subdomains) - and/or `url_surtkey` (SURT keys of URL patterns: `url_surtkey_prefixes` match all - keys starting with them, `url_surtkeys` exactly); predicates are OR-ed together. + domains with their subdomains (see domain_predicate) and/or `url_surtkey` (SURT keys + of URL patterns: `url_surtkey_prefixes` match all keys starting with them, + `url_surtkeys` exactly); predicates are OR-ed together. At least one host, domain or SURT key is required. If `crawls` is a non-empty list of crawl names (e.g. ['CC-MAIN-2025-33']), a `crawl IN (...)` partition filter is added -- the main lever for reducing scan cost. `conditions` are further (already @@ -80,7 +98,7 @@ def build_where_sql( query_tlds = ' OR '.join(f'url_host_tld = {escape_sql_literal(t)}' for t in sorted(tlds)) host_predicates = [f'url_host_name = {escape_sql_literal(h)}' for h in url_host_names] - host_predicates += [f'url_host_registered_domain = {escape_sql_literal(d)}' for d in domains] + host_predicates += [domain_predicate(d) for d in domains] host_predicates += [f'url_surtkey = {quote_sql_string(s)}' for s in surts] host_predicates += [f'url_surtkey {sql_like_prefix(s)}' for s in surt_prefixes] query_hosts = ' OR '.join(host_predicates) diff --git a/tests/repackage/test_athena_query_builder.py b/tests/repackage/test_athena_query_builder.py index dd3f2df..45878e0 100644 --- a/tests/repackage/test_athena_query_builder.py +++ b/tests/repackage/test_athena_query_builder.py @@ -45,7 +45,7 @@ def test_build_query_hostnames(): def test_build_query_domains_only(): q = build_athena_query(url_host_registered_domains=['example.com']) - assert "url_host_registered_domain = 'example.com'" in q + assert "url_host_registered_domain IN ('example.com')" in q assert "url_host_tld = 'com'" in q assert 'url_host_name' not in q @@ -53,7 +53,7 @@ def test_build_query_domains_only(): def test_build_query_hostnames_and_domains(): q = build_athena_query(['www.example.com'], url_host_registered_domains=['example.org']) assert "url_host_name = 'www.example.com'" in q - assert "url_host_registered_domain = 'example.org'" in q + assert "url_host_registered_domain IN ('example.org')" in q assert "url_host_tld = 'com'" in q assert "url_host_tld = 'org'" in q diff --git a/tests/unit/test_selectors.py b/tests/unit/test_selectors.py new file mode 100644 index 0000000..f76c2a8 --- /dev/null +++ b/tests/unit/test_selectors.py @@ -0,0 +1,71 @@ +"""The URL selectors select the same captures from every backend (CDX files, the columnar index with DuckDB).""" +import json + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.indexes.patterns import to_surt +from tests.conftest import requires_duckdb + + +# url, registered domain +CAPTURES = [ + ('https://example.com/', 'example.com'), + ('https://www.example.com/w', 'example.com'), + ('https://blog.example.com/', 'example.com'), + ('https://a.blog.example.com/', 'example.com'), + ('https://ablog.example.com/', 'example.com'), + ('https://blogexample.com/', 'blogexample.com'), + ('https://example.com.au/', 'example.com.au'), +] + + +@pytest.fixture(scope='module') +def indexes(tmp_path_factory): + """The same captures as a CDX file and as a columnar (parquet) index.""" + import duckdb + + base = tmp_path_factory.mktemp('indexes') + lines = sorted( + f'{to_surt(url)} 20240801120000 ' + f'{json.dumps({"url": url, "filename": "a.warc.gz", "offset": str(i), "length": "1"})}\n' + for i, (url, _) in enumerate(CAPTURES) + ) + (base / 'index.cdx').write_text(''.join(lines)) + + path = base / 'columnar' / 'crawl=CC-MAIN-2024-33' / 'subset=warc' + path.mkdir(parents=True) + values = ', '.join( + f"('{to_surt(url)}', '{url}', '{url.split('/')[2]}', '{domain}', '{domain.rsplit('.', 1)[-1]}', {i})" + for i, (url, domain) in enumerate(CAPTURES) + ) + duckdb.connect().execute(f''' + COPY (SELECT *, TIMESTAMP '2024-08-01 12:00:00' AS fetch_time, 'a.warc.gz' AS warc_filename, + 1 AS warc_record_length, NULL::VARCHAR AS content_mime_type, + NULL::VARCHAR AS content_mime_detected, NULL::SMALLINT AS fetch_status, + NULL::VARCHAR AS content_digest, NULL::VARCHAR AS content_languages, + NULL::VARCHAR AS content_charset, NULL::VARCHAR AS fetch_redirect + FROM (VALUES {values}) AS t(url_surtkey, url, url_host_name, url_host_registered_domain, url_host_tld, + warc_record_offset)) + TO '{path}/part-0.parquet' (FORMAT parquet) + ''') + return str(base / 'index.cdx'), str(base / 'columnar' / '*' / '*' / '*.parquet') + + +@requires_duckdb +@pytest.mark.parametrize('selector, expected', [ + (['--domain', 'example.com'], [ + 'https://a.blog.example.com/', 'https://ablog.example.com/', 'https://blog.example.com/', + 'https://example.com/', 'https://www.example.com/w', + ]), + (['--domain', 'blog.example.com'], ['https://a.blog.example.com/', 'https://blog.example.com/']), + (['--domain', 'a.blog.example.com'], ['https://a.blog.example.com/']), + (['--domain', 'example.com.au'], ['https://example.com.au/']), + (['--hostname', 'blog.example.com'], ['https://blog.example.com/']), + (['--url', '*.blog.example.com'], ['https://a.blog.example.com/', 'https://blog.example.com/']), + (['--url', 'blog.example.com/*'], ['https://blog.example.com/']), +]) +def test_selectors_match_the_same_captures_on_every_backend(indexes, capsys, selector, expected): + for index in indexes: + main(['index', index, '--yes', '--output-format', 'csv', '--fields', 'url'] + selector) + assert sorted(capsys.readouterr().out.splitlines()[1:]) == expected, index diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index c548d55..af58130 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -107,7 +107,9 @@ def test_duckdb_guided_query_prunes_crawls(): assert n_crawls == 1 assert "read_parquet(['s3://bucket/index/crawl=CC-MAIN-2024-33/subset=warc/*.parquet']" in sql assert 'crawl IN' not in sql # pruned by the glob - assert "url_host_name = 'www.example.com' OR url_host_registered_domain = 'example.org'" in sql + assert ("url_host_name = 'www.example.com' OR (url_host_registered_domain IN ('example.org') " + "AND (url_surtkey LIKE 'org,example)%' ESCAPE '\\' " + "OR url_surtkey LIKE 'org,example,%' ESCAPE '\\'))") in sql assert 'url_surtkey, fetch_time, url,' in sql assert sql.rstrip().endswith('LIMIT 5') @@ -117,7 +119,8 @@ def test_guided_query_url_patterns(): sql, _ = AthenaBackend(ATHENA).prepare(query) assert "(url_host_tld = 'com' OR url_host_tld = 'net' OR url_host_tld = 'org')" in sql - assert ("(url_host_registered_domain = 'example.com' OR url_surtkey = 'org,example)/' " + assert ("((url_host_registered_domain IN ('example.com') AND (url_surtkey LIKE 'com,example)%' ESCAPE '\\' " + "OR url_surtkey LIKE 'com,example,%' ESCAPE '\\')) OR url_surtkey = 'org,example)/' " "OR url_surtkey LIKE 'com,example)/a\\_b%' ESCAPE '\\' " "OR url_surtkey LIKE 'net,example)%' ESCAPE '\\' OR url_surtkey LIKE 'net,example,%' ESCAPE '\\')") in sql @@ -253,15 +256,15 @@ def test_index_duckdb_time_range_and_language(tmp_path, capsys): ('https://example.com/d', '2024-08-04 12:00:00', None), ] values = ', '.join( - f"('{url}', TIMESTAMPTZ '{t}+00', {repr(lang) if lang else 'NULL'}, 'example.com', 'example.com', 'com', " - f"'a.warc.gz', {i}, 100)" + f"('{url}', 'com,example)/{url[-1]}', TIMESTAMPTZ '{t}+00', {repr(lang) if lang else 'NULL'}, 'example.com', " + f"'example.com', 'com', 'a.warc.gz', {i}, 100)" for i, (url, t, lang) in enumerate(rows) ) duckdb.connect().execute(f''' - COPY (SELECT *, NULL::VARCHAR AS url_surtkey, NULL::VARCHAR AS content_mime_type, + COPY (SELECT *, NULL::VARCHAR AS content_mime_type, NULL::VARCHAR AS content_mime_detected, NULL::SMALLINT AS fetch_status, NULL::VARCHAR AS content_digest, NULL::VARCHAR AS content_charset, NULL::VARCHAR AS fetch_redirect - FROM (VALUES {values}) AS t(url, fetch_time, content_languages, url_host_registered_domain, + FROM (VALUES {values}) AS t(url, url_surtkey, fetch_time, content_languages, url_host_registered_domain, url_host_name, url_host_tld, warc_filename, warc_record_offset, warc_record_length)) TO '{path}/part-0.parquet' (FORMAT parquet) ''') From 4c122f2379b64b0b56ec20b614bb62e7f33dce1d Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:16:41 +0200 Subject: [PATCH 13/91] fix: overlapping CDX API selectors return each capture once The results of --url, --hostname and --domain were concatenated, so a capture selected by two of them was written twice and used up the limit twice. With several selectors, duplicates are now skipped (by URL, timestamp and WARC location, so distinct captures of the same URL stay), and --limit is applied after that instead of by the server. The help of --count says that the API estimate counts overlaps more than once. --- cdx_toolkit/cli/index.py | 4 ++-- cdx_toolkit/indexes/cdx_api.py | 21 ++++++++++++++++++-- tests/unit/test_indexes.py | 35 ++++++++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index 282941e..5329e9e 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -136,8 +136,8 @@ def add_parser(subparsers): ) output.add_argument( '--count', action='store_true', - help=('only print the number of captures (CDX API: cheap estimate based on the number of index pages; ' - 'SQL: exact count)'), + help=('only print the number of captures (CDX API: cheap estimate based on the number of index pages, ' + 'overlapping selectors are counted more than once; SQL: exact count)'), ) output.add_argument( '--dry-run', action='store_true', diff --git a/cdx_toolkit/indexes/cdx_api.py b/cdx_toolkit/indexes/cdx_api.py index 6de7ff3..a5b654c 100644 --- a/cdx_toolkit/indexes/cdx_api.py +++ b/cdx_toolkit/indexes/cdx_api.py @@ -33,13 +33,23 @@ def validate(self, query: IndexQuery) -> None: raise UsageError('--crawl can only be used with the Common Crawl index (cc)') def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: + """The captures of all selectors; a capture selected by more than one is only returned once. + + With several selectors, --limit is applied here (after removing the duplicates), not by the server.""" cdx = self._fetcher(query) + patterns = query.url_patterns() remaining = query.limit - for url, match_type in query.url_patterns(): - params = self._params(query, match_type, limit=remaining) + seen = set() if len(patterns) > 1 else None + for url, match_type in patterns: + params = self._params(query, match_type, limit=remaining if seen is None else None) # closest only makes sense for a single (non-paged) query captures = cdx.get(url, **params) if query.closest else cdx.iter(url, **params) for capture in captures: + if seen is not None: + identity = capture_identity(capture) + if identity in seen: + continue + seen.add(identity) yield capture if remaining is not None: remaining -= 1 @@ -47,6 +57,7 @@ def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: return def count(self, query: IndexQuery) -> int: + """Estimate from the number of index pages; overlapping selectors are counted more than once.""" cdx = self._fetcher(query) return sum( cdx.get_size_estimate(url, **self._params(query, match_type, limit=query.limit)) @@ -88,3 +99,9 @@ def _params(query: IndexQuery, match_type: Optional[str], limit: Optional[int]) if match_type: params['matchType'] = match_type return params + + +def capture_identity(capture: Mapping) -> tuple: + """What distinguishes captures: the same URL can be captured at the same time more than once (e.g. in + different crawls), so the WARC location is part of the identity.""" + return tuple(capture.get(field) for field in ('urlkey', 'url', 'timestamp', 'filename', 'offset', 'digest')) diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py index 366a049..c44ae3c 100644 --- a/tests/unit/test_indexes.py +++ b/tests/unit/test_indexes.py @@ -86,10 +86,41 @@ def test_cdx_api_url_patterns_and_limit(): calls = fetcher.return_value.iter.call_args_list assert len(calls) == 2 # the limit was reached before the domain query + # with several selectors, the limit is applied after removing duplicates, not by the server assert calls[0].args == ('example.com/*',) - assert calls[0].kwargs == {'limit': 3, 'from_ts': '2020', 'filter': ['=status:200']} + assert calls[0].kwargs == {'from_ts': '2020', 'filter': ['=status:200']} assert calls[1].args == ('www.example.org',) - assert calls[1].kwargs == {'limit': 1, 'from_ts': '2020', 'filter': ['=status:200'], 'matchType': 'host'} + assert calls[1].kwargs == {'from_ts': '2020', 'filter': ['=status:200'], 'matchType': 'host'} + + +def test_cdx_api_single_selector_limit_is_sent_to_the_server(): + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.iter.return_value = iter([_capture('http://example.com/a')]) + list(CdxApiBackend('cc').iter_records(IndexQuery(urls=['example.com/*'], limit=3))) + assert fetcher.return_value.iter.call_args.kwargs == {'limit': 3} + + +def test_cdx_api_overlapping_selectors_return_each_capture_once(): + def capture(url, timestamp, offset): + return {'url': url, 'timestamp': timestamp, 'filename': 'a.warc.gz', 'offset': offset} + + shared = capture('http://example.com/a', '20240101000000', '1') + again = capture('http://example.com/a', '20240201000000', '2') # same URL, another capture + other_crawl = capture('http://example.com/a', '20240101000000', '3') # same URL and time, another WARC record + results = { + 'example.com/*': [shared, again], + 'example.com': [dict(shared), other_crawl, capture('http://www.example.com/', '20240101000000', '4')], + } + query = IndexQuery(urls=['example.com/*'], domains=['example.com']) + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.iter.side_effect = lambda url, **params: iter(results[url]) + records = list(CdxApiBackend('cc').iter_records(query)) + assert [r['offset'] for r in records] == ['1', '2', '3', '4'] + + # the limit counts distinct captures + query.limit = 3 + records = list(CdxApiBackend('cc').iter_records(query)) + assert [r['offset'] for r in records] == ['1', '2', '3'] def test_cdx_api_closest_uses_get_and_crawl_is_normalized(): From 4a6c2ce80c021c3177c6a8676abe286c33b0c1a9 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:19:04 +0200 Subject: [PATCH 14/91] fix: --limit applies before the URL filters for CDX files, --count is capped by --limit - CDX files applied --url-fgrep/--url-fgrepv while reading, before --limit, contrary to the documented order; they are now applied after the limit like for the other backends (one output per CDX file, which rejects --limit, still filters while reading) - --count is at most --limit on every backend (the CDX API estimate and the CDX file count ignored it) - --count with --url-fgrep/--url-fgrepv is a usage error (they are client-side filters) - --limit below 1 is a usage error --- cdx_toolkit/cli/index.py | 6 ++++-- cdx_toolkit/indexes/base.py | 5 ++++- cdx_toolkit/indexes/cdx_api.py | 6 ++++-- cdx_toolkit/indexes/cdx_files.py | 15 ++++++++------- tests/unit/test_cdx_files.py | 24 +++++++++++++++++++++--- tests/unit/test_indexes.py | 12 +++++++----- tests/unit/test_selectors.py | 9 +++++++++ 7 files changed, 57 insertions(+), 20 deletions(-) diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index 5329e9e..a952dc2 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -136,8 +136,8 @@ def add_parser(subparsers): ) output.add_argument( '--count', action='store_true', - help=('only print the number of captures (CDX API: cheap estimate based on the number of index pages, ' - 'overlapping selectors are counted more than once; SQL: exact count)'), + help=('only print the number of captures, at most --limit (CDX API: cheap estimate based on the number of ' + 'index pages, overlapping selectors are counted more than once; CDX files and SQL: exact count)'), ) output.add_argument( '--dry-run', action='store_true', @@ -243,6 +243,8 @@ def run(args: Namespace, cmdline: str): return if args.count: + if query.url_fgrep or query.url_fgrepv: + raise UsageError('--count cannot be used with --url-fgrep or --url-fgrepv') count = backend.count(query) with open_text(args.output, 'w') as fh: fh.write(f'{count}\n') diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py index 8e5cbcf..479c6b3 100644 --- a/cdx_toolkit/indexes/base.py +++ b/cdx_toolkit/indexes/base.py @@ -25,6 +25,8 @@ def validate(self, query: IndexQuery) -> None: if unsupported: options = ', '.join(field_flag(option) for option in unsupported) raise UsageError(f'{options} cannot be used with the {self.name} index') + if query.limit is not None and query.limit < 1: + raise UsageError(f'--limit must be at least 1, not {query.limit}') @abstractmethod def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: @@ -35,7 +37,8 @@ def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: @abstractmethod def count(self, query: IndexQuery) -> int: - """Cheap (possibly estimated) number of captures matching the query.""" + """Cheap (possibly estimated) number of captures matching the query, at most --limit (without the + client-side --url-fgrep/--url-fgrepv, which cannot be combined with --count).""" raise NotImplementedError # pragma: no cover @abstractmethod diff --git a/cdx_toolkit/indexes/cdx_api.py b/cdx_toolkit/indexes/cdx_api.py index a5b654c..84db22a 100644 --- a/cdx_toolkit/indexes/cdx_api.py +++ b/cdx_toolkit/indexes/cdx_api.py @@ -57,12 +57,14 @@ def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: return def count(self, query: IndexQuery) -> int: - """Estimate from the number of index pages; overlapping selectors are counted more than once.""" + """Estimate from the number of index pages, at most --limit; overlapping selectors are counted more + than once.""" cdx = self._fetcher(query) - return sum( + count = sum( cdx.get_size_estimate(url, **self._params(query, match_type, limit=query.limit)) for url, match_type in query.url_patterns() ) + return min(count, query.limit) if query.limit else count def explain(self, query: IndexQuery) -> str: lines = [] diff --git a/cdx_toolkit/indexes/cdx_files.py b/cdx_toolkit/indexes/cdx_files.py index 10d5eb7..6e00a7d 100644 --- a/cdx_toolkit/indexes/cdx_files.py +++ b/cdx_toolkit/indexes/cdx_files.py @@ -65,7 +65,9 @@ class CaptureFilter: url_fgrepv: Optional[str] = None @classmethod - def from_query(cls, query: IndexQuery) -> 'CaptureFilter': + def from_query(cls, query: IndexQuery, url_filters: bool = False) -> 'CaptureFilter': + """The selection of the query; --url-fgrep/--url-fgrepv only with `url_filters` (one output per file: + without --limit, they can be applied while reading; otherwise they are applied after the limit).""" prefixes, exact = [], [] patterns = list(query.urls) if query.url_file: @@ -90,8 +92,8 @@ def from_query(cls, query: IndexQuery) -> 'CaptureFilter': select_all=not (query.urls or query.url_file or query.hostnames or query.domains), from_ts=pad_timestamp(query.from_ts) if query.from_ts else None, to=pad_timestamp_up(query.to) if query.to else None, - url_fgrep=query.url_fgrep, - url_fgrepv=query.url_fgrepv, + url_fgrep=query.url_fgrep if url_filters else None, + url_fgrepv=query.url_fgrepv if url_filters else None, ) def matcher(self) -> 'LineMatcher': @@ -216,9 +218,8 @@ def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: return def count(self, query: IndexQuery) -> int: - """Exact number of selected captures (reads all files).""" - matcher = CaptureFilter.from_query(query).matcher() - return sum(1 for path, _ in self.files() for _ in iter_matching_lines(path, matcher)) + """Exact number of selected captures, at most --limit (reads all files, or until the limit).""" + return sum(1 for _ in self.iter_records(query)) def explain(self, query: IndexQuery) -> str: capture_filter = CaptureFilter.from_query(query) @@ -238,7 +239,7 @@ def write_per_file(self, query: IndexQuery, output_base: str, processes: int = 1 Returns the number of output files and captures; no file is written without matches.""" if query.limit: raise UsageError('--limit cannot be used when writing one output per CDX file') - capture_filter = CaptureFilter.from_query(query) + capture_filter = CaptureFilter.from_query(query, url_filters=True) join = (lambda rel: output_base.rstrip('/') + '/' + rel) if '://' in output_base else \ (lambda rel: os.path.join(output_base, *rel.split('/'))) tasks = [(path, join(rel), capture_filter) for path, rel in self.files()] diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py index 4798a94..cc4c2ee 100644 --- a/tests/unit/test_cdx_files.py +++ b/tests/unit/test_cdx_files.py @@ -5,7 +5,7 @@ import pytest from cdx_toolkit.cli import main -from cdx_toolkit.indexes import IndexQuery, UsageError +from cdx_toolkit.indexes import IndexQuery, UsageError, iter_index_records from cdx_toolkit.indexes.cdx_files import ( CdxFilesBackend, domain_entry_to_surts, load_url_patterns, url_pattern_to_surts, ) @@ -61,12 +61,27 @@ def test_time_range_fgrep_and_limit(): query = IndexQuery(domains=['missiondefrance.fr'], from_ts='20240716', to='20240716') assert backend.count(query) == count_lines('fr,missiondefrance,', day='20240716') - records = list(backend.iter_records(IndexQuery(domains=['mnhn.fr'], url_fgrep='/taxonomy/', limit=5))) - assert len(records) == 5 + records = list(iter_index_records(backend, IndexQuery(domains=['mnhn.fr'], url_fgrep='/taxonomy/', limit=5))) + assert 0 < len(records) <= 5 assert all('/taxonomy/' in r['url'] and r['urlkey'].startswith('fr,mnhn,biodiv)') for r in records) assert set(records[0]) >= {'urlkey', 'timestamp', 'url', 'filename', 'offset', 'length'} +def test_limit_applies_before_the_url_filters(tmp_path, capsys): + path = tmp_path / 'index.cdx' + path.write_text(''.join( + f'com,example)/{name} 20240101000000 {json.dumps({"url": f"https://example.com/{name}"})}\n' + for name in ('drop', 'keep') + )) + backend = CdxFilesBackend(str(path)) + query = IndexQuery(domains=['example.com'], limit=1, url_fgrep='keep') + assert list(iter_index_records(backend, query)) == [] + query.url_fgrep, query.url_fgrepv = None, 'drop' + assert list(iter_index_records(backend, query)) == [] + assert backend.count(IndexQuery(domains=['example.com'], limit=1)) == 1 + assert backend.count(IndexQuery(domains=['example.com'])) == 2 + + def test_url_file_formats(tmp_path): urls = ['*.missiondefrance.fr', 'biodiv.mnhn.fr/fr/taxonomy*'] txt = tmp_path / 'urls.txt' @@ -126,6 +141,9 @@ def test_cli_combined_output_and_errors(tmp_path, capsys): (['--domain', 'mnhn.fr', '-o', str(tmp_path / 'out'), '--limit', '1'], '--limit cannot be used'), (['--domain', 'mnhn.fr', '-o', str(tmp_path / 'out'), '--fields', 'url'], '--fields cannot be used'), (['--url-column', 'x', '--domain', 'mnhn.fr'], '--url-column and --url-file-type need --url-file'), + (['--domain', 'mnhn.fr', '--limit', '0'], '--limit must be at least 1'), + (['--domain', 'mnhn.fr', '--limit', '-1'], '--limit must be at least 1'), + (['--domain', 'mnhn.fr', '--count', '--url-fgrep', 'x'], '--count cannot be used with --url-fgrep'), ]: with pytest.raises(SystemExit): main(['index', str(CDX_PATH)] + args) diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py index c44ae3c..e1ab59a 100644 --- a/tests/unit/test_indexes.py +++ b/tests/unit/test_indexes.py @@ -139,12 +139,14 @@ def test_cdx_api_count(): query = IndexQuery(urls=['example.com/*'], domains=['example.org'], limit=10) with patch('cdx_toolkit.CDXFetcher') as fetcher: - fetcher.return_value.get_size_estimate.side_effect = [3000, 1500] - assert CdxApiBackend('ia').count(query) == 4500 + fetcher.return_value.get_size_estimate.side_effect = [3000, 1500, 3000, 1500] + assert CdxApiBackend('ia').count(query) == 10 # capped by the limit + calls = fetcher.return_value.get_size_estimate.call_args_list + assert calls[1].args == ('example.org',) + assert calls[1].kwargs == {'limit': 10, 'matchType': 'domain'} - calls = fetcher.return_value.get_size_estimate.call_args_list - assert calls[1].args == ('example.org',) - assert calls[1].kwargs == {'limit': 10, 'matchType': 'domain'} + query.limit = None + assert CdxApiBackend('ia').count(query) == 4500 def test_iter_index_records_url_fgrep(): diff --git a/tests/unit/test_selectors.py b/tests/unit/test_selectors.py index f76c2a8..e076d76 100644 --- a/tests/unit/test_selectors.py +++ b/tests/unit/test_selectors.py @@ -69,3 +69,12 @@ def test_selectors_match_the_same_captures_on_every_backend(indexes, capsys, sel for index in indexes: main(['index', index, '--yes', '--output-format', 'csv', '--fields', 'url'] + selector) assert sorted(capsys.readouterr().out.splitlines()[1:]) == expected, index + + +@requires_duckdb +def test_count_is_capped_by_the_limit_on_every_backend(indexes, capsys): + for index in indexes: + for limit, expected in ((None, '5'), ('2', '2'), ('9', '5')): + limit_args = ['--limit', limit] if limit else [] + main(['index', index, '--yes', '--domain', 'example.com', '--count'] + limit_args) + assert capsys.readouterr().out.strip() == expected, (index, limit) From a418826be07cca6a65fa8644a3fb42cf7c771bcf Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:21:26 +0200 Subject: [PATCH 15/91] fix: HTTP range reads of parallel readers run concurrently The HTTP reader called the synchronous request helper from the event loop, so network waits and retry sleeps blocked the other readers and the writer, and --parallel-readers had no effect for http(s). The request now runs in a worker thread. The minimum interval between requests to a host (e.g. 0.55s for data.commoncrawl.org) is kept across threads: each request reserves its start time under a lock. --- cdx_toolkit/myrequests.py | 34 ++++++++++++++++---- cdx_toolkit/repackage/data_classes.py | 5 ++- tests/repackage/test_warc_filter_pipeline.py | 25 ++++++++++++++ tests/unit/test_myrequests.py | 17 ++++++++++ 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/cdx_toolkit/myrequests.py b/cdx_toolkit/myrequests.py index 6e8d684..85c0e30 100644 --- a/cdx_toolkit/myrequests.py +++ b/cdx_toolkit/myrequests.py @@ -1,6 +1,7 @@ from typing import Optional import requests import logging +import threading import time from urllib.parse import urlparse @@ -63,6 +64,28 @@ def update_next_fetch(hostname, next_fetch): retry_info[hostname]['next_fetch'] = next_fetch +# requests may be made from several threads (e.g. the parallel WARC readers of cdxt repackage) +_retry_lock = threading.Lock() + + +def reserve_fetch_time(hostname: str, now: float) -> float: + """The time at which the next request to `hostname` may start, at least the host's minimum interval + after the previous one; the time is reserved, so concurrent requests are spaced out too.""" + with _retry_lock: + next_fetch, minimum_interval = get_retries(hostname) + fetch_time = max(now, next_fetch) + update_next_fetch(hostname, fetch_time + minimum_interval) + return fetch_time + + +def finish_fetch(hostname: str) -> None: + """After a request: the next one may start the minimum interval from now (or later, if reserved).""" + with _retry_lock: + get_retries(hostname) + entry = retry_info[hostname] + entry['next_fetch'] = max(entry['next_fetch'], time.time() + entry['minimum_interval']) + + def myrequests_get( url, params=None, @@ -82,15 +105,14 @@ def myrequests_get( raise_warning_after_n_errors = WARNING_AFTER_N_ERRORS hostname = urlparse(url).hostname - next_fetch, minimum_interval = get_retries(hostname) + fetch_time = reserve_fetch_time(hostname, t) # next_fetch is also updated at the bottom + minimum_interval = retry_info[hostname]['minimum_interval'] - if t < next_fetch: - dt = next_fetch - t + if t < fetch_time: + dt = fetch_time - t if dt > 3.1: LOGGER.debug('sleeping for {:.3f}s before next fetch'.format(dt)) time.sleep(dt) - # next_fetch is also updated at the bottom - update_next_fetch(hostname, next_fetch + minimum_interval) if params: if 'from_ts' in params: @@ -176,6 +198,6 @@ def myrequests_get( previously_seen_hostnames.add(hostname) # in case we had a lot of retries, etc - update_next_fetch(hostname, time.time() + minimum_interval) + finish_fetch(hostname) return resp diff --git a/cdx_toolkit/repackage/data_classes.py b/cdx_toolkit/repackage/data_classes.py index 9e66f4f..3e897cd 100644 --- a/cdx_toolkit/repackage/data_classes.py +++ b/cdx_toolkit/repackage/data_classes.py @@ -1,3 +1,4 @@ +import asyncio import time from dataclasses import dataclass, field @@ -130,7 +131,9 @@ async def ranged_get_bytes( # read from HTTP headers = {'Range': 'bytes={}-{}'.format(offset, end)} - resp = myrequests_get(self.url, headers=headers) + # in a thread, so that the event loop (other readers, the writer) is not blocked; myrequests_get keeps + # the minimum interval between requests to a host across threads + resp = await asyncio.to_thread(myrequests_get, self.url, headers=headers) check_range_response( resp.status_code, resp.headers.get('Content-Range'), len(resp.content), offset, length, self.url ) diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 3c7f79f..6962bca 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -2,6 +2,7 @@ import asyncio import contextlib import glob +import gzip import itertools import os import threading @@ -75,6 +76,30 @@ def test_invalid_range_responses_are_not_written(tmp_path, response): assert glob.glob(str(tmp_path / 'out-*')) == [] +def test_a_slow_http_read_does_not_block_the_other_readers_and_the_writer(tmp_path): + """The first read is held until the second capture has been written.""" + second_written = threading.Event() + original_write = LocalFileWriter.write + + def get(url, headers=None, **kwargs): + if url.endswith(CAPTURES[0]['filename']): + assert second_written.wait(timeout=5), 'the second capture was not written while the first was read' + return fake_get(url, headers) + + async def write(self, data): + await original_write(self, data) + if b'https://example.com/1' in gzip.decompress(data): + second_written.set() + + wf = WARCFilter(source=RecordsSource(CAPTURES[:2], 'https://data.example.org', sort=False), + prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=2) + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=get), \ + patch.object(LocalFileWriter, 'write', write): + assert wf.filter() == RepackageResult(requested=2, written=2) + records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) + assert [r.rec_headers.get_header('WARC-Target-URI') for r in records] == [CAPTURES[1]['url'], CAPTURES[0]['url']] + + def test_writer_error_stops_the_pipeline_and_removes_the_file(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, max_file_size=1) diff --git a/tests/unit/test_myrequests.py b/tests/unit/test_myrequests.py index 588dc4f..fede3cb 100644 --- a/tests/unit/test_myrequests.py +++ b/tests/unit/test_myrequests.py @@ -25,3 +25,20 @@ def test_get_with_connection_error(caplog): log_levels = [r.levelname for r in caplog.records] assert log_levels.count('WARNING') == 2 assert log_levels.count('ERROR') == 1 + + +def test_concurrent_requests_keep_the_minimum_interval(): + import threading + + from cdx_toolkit import myrequests + + host = 'interval-test.example.org' + myrequests.retry_info[host] = {'next_fetch': 1.0, 'minimum_interval': 0.5} + times = [] + threads = [threading.Thread(target=lambda: times.append(myrequests.reserve_fetch_time(host, 1000.0))) + for _ in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert sorted(times) == [1000.0, 1000.5, 1001.0, 1001.5] From 39ec559de8d5e472a42142b532d82aee6031e133 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:23:00 +0200 Subject: [PATCH 16/91] fix: explicit AWS settings win over the defaults: AWS_REGION and retry bounds - botocore ignores AWS_REGION, so with only AWS_REGION set the client had no region (or the default); it is now passed to botocore explicitly (an explicit region still wins, then AWS_REGION, AWS_DEFAULT_REGION, the profile, the default) - an explicit retries.max_attempts (e.g. in the fsspec configuration) was merged with the default total_max_attempts=100000, which botocore prefers; an explicit bound of either form now replaces the default one The tests check the configuration of real botocore clients (no requests). --- cdx_toolkit/aws.py | 47 ++++++++++++++++++++++++++----------- tests/unit/test_aws.py | 53 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 14 deletions(-) diff --git a/cdx_toolkit/aws.py b/cdx_toolkit/aws.py index 9d95f3d..9d37c43 100644 --- a/cdx_toolkit/aws.py +++ b/cdx_toolkit/aws.py @@ -11,8 +11,10 @@ (multipart_threshold = multipart_chunksize = 10GB: no multipart transfers) The user stays in control: a default is only used for a setting that is not configured in the -environment (AWS_REGION / AWS_DEFAULT_REGION, AWS_RETRY_MODE, AWS_MAX_ATTEMPTS) or in the AWS config -file of the active profile (~/.aws/config, AWS_PROFILE). CDXT_AWS_DEFAULTS=0 disables all defaults. +environment (AWS_REGION / AWS_DEFAULT_REGION, AWS_RETRY_MODE, AWS_MAX_ATTEMPTS), in the AWS config +file of the active profile (~/.aws/config, AWS_PROFILE) or explicitly (e.g. the retries in the fsspec +configuration). CDXT_AWS_DEFAULTS=0 disables all defaults. The region is, in this order: an explicit +region, AWS_REGION (which botocore itself ignores), AWS_DEFAULT_REGION, the profile, the default. The multipart settings are for whole-file transfers (`aws s3 cp`): cdxt reads WARC records with one range request each and reads index files sequentially, so it makes no multipart downloads anyway. @@ -34,7 +36,7 @@ DEFAULT_S3 = {'addressing_style': 'virtual', 'payload_signing_enabled': False} DEFAULT_MAX_CONCURRENT_REQUESTS = 2 -# environment variables that configure a setting (botocore reads them itself) +# environment variables that configure a setting (botocore reads them itself, except AWS_REGION) SETTING_ENV_VARS = { 'region': ('AWS_REGION', 'AWS_DEFAULT_REGION'), 'retry_mode': ('AWS_RETRY_MODE',), @@ -69,14 +71,20 @@ def user_s3_setting(name: str): return s3.get(name) if isinstance(s3, dict) else None +# the retry settings that bound the number of attempts (botocore prefers total_max_attempts) +RETRY_BOUNDS = ('max_attempts', 'total_max_attempts') + + def botocore_config_kwargs() -> dict: """kwargs for botocore.config.Config with the defaults for the settings the user did not configure. botocore merges the `retries` and `s3` settings key by key with the environment and the AWS config file, so the settings the user configured keep their values.""" - if not defaults_enabled(): - return {} kwargs = {} + if os.environ.get('AWS_REGION'): + kwargs['region_name'] = os.environ['AWS_REGION'] # botocore only reads AWS_DEFAULT_REGION + if not defaults_enabled(): + return kwargs if not user_configured('region'): kwargs['region_name'] = DEFAULT_REGION retries = {} @@ -92,16 +100,32 @@ def botocore_config_kwargs() -> dict: return kwargs +def merge_config_kwargs(defaults: dict, explicit: dict) -> dict: + """Explicit botocore config kwargs win over the defaults; the `retries` and `s3` dicts are merged key by key, + and an explicit bound of the attempts (max_attempts or total_max_attempts) replaces the default one.""" + merged = dict(defaults) + for key, value in explicit.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + default = dict(merged[key]) + if key == 'retries' and any(bound in value for bound in RETRY_BOUNDS): + for bound in RETRY_BOUNDS: + default.pop(bound, None) + merged[key] = {**default, **value} + else: + merged[key] = value + return merged + + def botocore_config(region_name: Optional[str] = None, **kwargs): - """A botocore Config with the defaults, plus the given (non-default) options, e.g. timeouts. + """A botocore Config with the defaults, plus the given options, e.g. timeouts (they win over the defaults). An explicit `region_name` wins over the default and the user's configuration.""" from botocore.config import Config - config_kwargs = botocore_config_kwargs() + config_kwargs = merge_config_kwargs(botocore_config_kwargs(), kwargs) if region_name: config_kwargs['region_name'] = region_name - return Config(**config_kwargs, **kwargs) + return Config(**config_kwargs) def max_concurrent_requests() -> int: @@ -119,12 +143,7 @@ def configure_fsspec_s3() -> None: import fsspec.config conf = fsspec.config.conf.setdefault('s3', {}) - config_kwargs = dict(conf.get('config_kwargs') or {}) - for key, value in botocore_config_kwargs().items(): - if isinstance(value, dict) and isinstance(config_kwargs.get(key), dict): - config_kwargs[key] = {**value, **config_kwargs[key]} - else: - config_kwargs.setdefault(key, value) + config_kwargs = merge_config_kwargs(botocore_config_kwargs(), dict(conf.get('config_kwargs') or {})) if config_kwargs: conf['config_kwargs'] = config_kwargs conf.setdefault('max_concurrency', max_concurrent_requests()) diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py index 59d1fb7..3623651 100644 --- a/tests/unit/test_aws.py +++ b/tests/unit/test_aws.py @@ -184,3 +184,56 @@ def test_invalid_region(no_user_config, monkeypatch): aws.aws_region() with pytest.raises(ValueError, match='invalid AWS region'): aws.duckdb_settings() + + +def s3_client(config=None): + """A real botocore S3 client (no requests are sent), to check the configuration botocore resolves.""" + import botocore.session + + session = botocore.session.Session() + return session.create_client('s3', config=config, aws_access_key_id='x', aws_secret_access_key='x') + + +@pytest.mark.parametrize('env, region', [ + ({}, 'us-east-1'), + ({'AWS_REGION': 'eu-west-1'}, 'eu-west-1'), + ({'AWS_DEFAULT_REGION': 'eu-central-1'}, 'eu-central-1'), + ({'AWS_REGION': 'eu-west-1', 'AWS_DEFAULT_REGION': 'eu-central-1'}, 'eu-west-1'), + ({'AWS_REGION': 'eu-west-1', 'CDXT_AWS_DEFAULTS': '0'}, 'eu-west-1'), +]) +def test_effective_region(no_user_config, monkeypatch, env, region): + pytest.importorskip('botocore') + for name, value in env.items(): + monkeypatch.setenv(name, value) + assert s3_client(aws.botocore_config()).meta.region_name == region + # an explicit region wins + assert s3_client(aws.botocore_config(region_name='ap-south-1')).meta.region_name == 'ap-south-1' + + +def test_effective_region_of_the_profile(no_user_config, monkeypatch, tmp_path): + pytest.importorskip('botocore') + (tmp_path / 'aws-config').write_text('[default]\nregion = sa-east-1\n') + with patch.object(aws, 'user_aws_config', return_value={'region': 'sa-east-1'}): + assert s3_client(aws.botocore_config()).meta.region_name == 'sa-east-1' + + +@pytest.mark.parametrize('retries, expected', [ + ({'max_attempts': 3}, {'mode': 'adaptive', 'total_max_attempts': 4}), # botocore: max_attempts = retries + ({'total_max_attempts': 3}, {'mode': 'adaptive', 'total_max_attempts': 3}), + ({'mode': 'standard'}, {'mode': 'standard', 'total_max_attempts': 100000}), +]) +def test_effective_explicit_retries(no_user_config, retries, expected): + pytest.importorskip('botocore') + assert s3_client(aws.botocore_config(retries=retries)).meta.config.retries == expected + + +@pytest.mark.parametrize('retries, expected', [ + ({'max_attempts': 3}, {'mode': 'adaptive', 'max_attempts': 3}), + ({'total_max_attempts': 3}, {'mode': 'adaptive', 'total_max_attempts': 3}), +]) +def test_fsspec_retries_of_the_user_win(no_user_config, monkeypatch, retries, expected): + import fsspec.config + + monkeypatch.setattr(fsspec.config, 'conf', {'s3': {'config_kwargs': {'retries': retries}}}) + aws.configure_fsspec_s3() + assert fsspec.config.conf['s3']['config_kwargs']['retries'] == expected From 81b75981439af2a28585294c0428f9174f1bcac6 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:24:12 +0200 Subject: [PATCH 17/91] fix: SQL --to includes the fractions of its last second --to 20240801 became fetch_time <= '2024-08-01 23:59:59', which excluded captures at e.g. 23:59:59.500. The end is now exclusive at the next second (fetch_time < '2024-08-02 00:00:00'); an end beyond year 9999 is a usage error. --- cdx_toolkit/indexes/sql.py | 13 ++++++++----- tests/unit/test_sql_indexes.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/cdx_toolkit/indexes/sql.py b/cdx_toolkit/indexes/sql.py index f7f2141..b6ebd4b 100644 --- a/cdx_toolkit/indexes/sql.py +++ b/cdx_toolkit/indexes/sql.py @@ -101,14 +101,17 @@ def normalize_row(row: Mapping) -> dict: return record -def sql_timestamp(timestamp: str, up: bool = False) -> str: - """CDX timestamp (e.g. 2008, 200906302359) -> SQL TIMESTAMP literal of its first (or last) second.""" +def sql_timestamp(timestamp: str, end: bool = False) -> str: + """CDX timestamp (e.g. 2008, 200906302359) -> SQL TIMESTAMP literal of its first second, or with `end`, of the + second after its last second (an exclusive end, which includes the fractions of the last second).""" if not re.fullmatch(r'\d{4,14}', timestamp): raise UsageError(f'invalid timestamp {timestamp!r}, use e.g. 2008, 20080328 or 200906302359') - padded = pad_timestamp_up(timestamp) if up else pad_timestamp(timestamp) + padded = pad_timestamp_up(timestamp) if end else pad_timestamp(timestamp) try: moment = datetime.datetime.strptime(padded, '%Y%m%d%H%M%S') - except ValueError: + if end: + moment += datetime.timedelta(seconds=1) + except (ValueError, OverflowError): raise UsageError(f'invalid timestamp {timestamp!r}') from None return f"TIMESTAMP '{moment:%Y-%m-%d %H:%M:%S}'" @@ -137,7 +140,7 @@ def query_conditions(query: IndexQuery) -> List[str]: if query.from_ts: conditions.append(f'fetch_time >= {sql_timestamp(query.from_ts)}') if query.to: - conditions.append(f'fetch_time <= {sql_timestamp(query.to, up=True)}') + conditions.append(f'fetch_time < {sql_timestamp(query.to, end=True)}') if query.languages: # primary content language: the first of the comma-separated content_languages predicates = [] diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index af58130..c7355cd 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -8,7 +8,9 @@ from cdx_toolkit.indexes import IndexQuery, UsageError, resolve_index from cdx_toolkit.indexes.cdx_api import CdxApiBackend from cdx_toolkit.indexes.cdx_files import CdxFilesBackend -from cdx_toolkit.indexes.sql import AthenaBackend, check_language, DuckDbBackend, normalize_row, parse_athena_location +from cdx_toolkit.indexes.sql import ( + AthenaBackend, check_language, DuckDbBackend, normalize_row, parse_athena_location, query_conditions, +) from tests.conftest import requires_duckdb @@ -130,7 +132,7 @@ def test_guided_query_time_range_and_languages(): sql, _ = AthenaBackend(ATHENA).prepare(query) assert "fetch_time >= TIMESTAMP '2024-01-01 00:00:00'" in sql - assert "fetch_time <= TIMESTAMP '2024-04-10 23:59:59'" in sql + assert "fetch_time < TIMESTAMP '2024-04-11 00:00:00'" in sql assert ("(content_languages = 'deu' OR content_languages LIKE 'deu,%' " "OR content_languages = 'fra' OR content_languages LIKE 'fra,%')") in sql @@ -336,3 +338,26 @@ def test_sql_without_url_column_to_repackage(parquet_index, tmp_path, capsys): main(['index', files, '--sql', str(sql), '--yes', '--url-fgrep', 'example']) assert exc.value.code == 2 assert '--url-fgrep and --url-fgrepv need captures with a url' in capsys.readouterr().err + + +@requires_duckdb +@pytest.mark.parametrize('to, included', [ + ('20240801', ['23:59:59.500']), + ('20240801235959', ['23:59:59.500']), + ('2024080123595', ['23:59:59.500']), + ('20240802', ['23:59:59.500', 'next day']), +]) +def test_end_timestamp_includes_fractional_seconds(to, included): + import duckdb + + rows = {'23:59:59.500': '2024-08-01 23:59:59.500', 'next day': '2024-08-02 00:00:00'} + con = duckdb.connect() + con.execute('CREATE TABLE t (name VARCHAR, fetch_time TIMESTAMP)') + con.executemany('INSERT INTO t VALUES (?, ?)', list(rows.items())) + condition, = query_conditions(IndexQuery(to=to)) + assert sorted(r[0] for r in con.execute(f'SELECT name FROM t WHERE {condition}').fetchall()) == included + + +def test_end_timestamp_overflow(): + with pytest.raises(UsageError, match='invalid timestamp'): + query_conditions(IndexQuery(to='9999')) From 8488edd4c8f6216071a0f3c4411ce8fd09ce1059 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:25:03 +0200 Subject: [PATCH 18/91] fix: WARC files staged for hf:// (and other non-S3 remote) output name themselves The files were staged as warc-001.warc.gz and renamed when copied, so their warcinfo record named the staging file. They are now staged under the basename of the prefix, so the WARC-Filename is the final name from the start. --- cdx_toolkit/repackage/multiprocess.py | 4 +++- tests/repackage/test_warc_filter_pipeline.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index 51076bf..d5ecfa6 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -26,6 +26,7 @@ import logging import multiprocessing import os +import posixpath import shutil import tempfile import time @@ -146,7 +147,8 @@ def run_multiprocess_repackage( staging_dir = os.path.join(tmp_dir, 'out') if remote_non_s3: os.makedirs(staging_dir) - write_prefix = os.path.join(staging_dir, 'warc') if remote_non_s3 else prefix_path + # staged under the final file names, so that the warcinfo records name the files they are in + write_prefix = os.path.join(staging_dir, posixpath.basename(prefix_path)) if remote_non_s3 else prefix_path configs = [ dict( diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 6962bca..f56f1d9 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -300,6 +300,10 @@ def test_multiprocess_repackage(tmp_path, remote): # one file per record (--size 1), numbered by the shared counter assert sorted(os.path.basename(p) for p in paths) == [f'RUN-SUB-00{i}.warc.gz' for i in range(1, 5)] assert len(response_records(paths)) == 4 + for path in paths: # the warcinfo record names the file it is in + with fsspec.open(path, 'rb') as f: + warcinfo = next(iter(ArchiveIterator(f))) + assert warcinfo.rec_headers.get_header('WARC-Filename') == os.path.basename(path) finally: if remote: fs.rm('/mp-test', recursive=True) From 6a592e7c4cd5199e57352eaf859cb5d530b0845f Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:28:18 +0200 Subject: [PATCH 19/91] refactor: SQL query building and execution move to indexes, repackage reads index records only - the SQL query building (escaping, guided WHERE clause, crawl names) is now cdx_toolkit.indexes.sql_query; running queries is cdx_toolkit.indexes.athena and cdx_toolkit.indexes.duckdb, which yield rows; the range-job paths of the Athena and DuckDB sources, build_athena_query and validate_result_columns are gone (cdxt index writes the rows, cdxt repackage reads them like any index file) - confirm_cost takes the number of crawls and the engine (no CostEstimate type, no dependency of indexes on repackage); the unused SqlBackend._prepared is gone - repackage/sources/ is now repackage/records_source.py (RecordsSource, join_warc_url); the multi-process shards are JSONL files written and read with record_io instead of a separate range-job CSV reader/writer; RangeJob.extra is gone --- cdx_toolkit/cli/repackage.py | 2 +- .../{repackage/sources => indexes}/athena.py | 97 ++------ cdx_toolkit/indexes/duckdb.py | 80 +++++++ cdx_toolkit/indexes/sql.py | 42 ++-- .../sql_base.py => indexes/sql_query.py} | 55 +---- cdx_toolkit/repackage/data_classes.py | 11 +- cdx_toolkit/repackage/multiprocess.py | 31 +-- .../{sources/records.py => records_source.py} | 22 +- cdx_toolkit/repackage/sources/__init__.py | 3 - cdx_toolkit/repackage/sources/base.py | 30 --- cdx_toolkit/repackage/sources/csv.py | 120 ---------- cdx_toolkit/repackage/sources/duckdb.py | 107 --------- cdx_toolkit/repackage/warc_filter.py | 5 +- docs/notes/warc-fetcher-performance.md | 2 +- tests/repackage/test_athena_query_builder.py | 223 ------------------ tests/repackage/test_csv_source.py | 130 ---------- tests/repackage/test_records_source.py | 26 ++ tests/repackage/test_warc_filter.py | 2 +- tests/repackage/test_warc_filter_pipeline.py | 2 +- .../test_athena.py} | 84 +++++-- tests/unit/test_aws.py | 2 +- .../test_confirm_cost.py} | 24 +- .../test_duckdb.py} | 33 +-- tests/unit/test_sql_query.py | 81 +++++++ tests/unit/test_usage_errors.py | 2 +- 25 files changed, 358 insertions(+), 858 deletions(-) rename cdx_toolkit/{repackage/sources => indexes}/athena.py (59%) create mode 100644 cdx_toolkit/indexes/duckdb.py rename cdx_toolkit/{repackage/sources/sql_base.py => indexes/sql_query.py} (79%) rename cdx_toolkit/repackage/{sources/records.py => records_source.py} (66%) delete mode 100644 cdx_toolkit/repackage/sources/__init__.py delete mode 100644 cdx_toolkit/repackage/sources/base.py delete mode 100644 cdx_toolkit/repackage/sources/csv.py delete mode 100644 cdx_toolkit/repackage/sources/duckdb.py delete mode 100644 tests/repackage/test_athena_query_builder.py delete mode 100644 tests/repackage/test_csv_source.py create mode 100644 tests/repackage/test_records_source.py rename tests/{repackage/test_athena_source.py => unit/test_athena.py} (66%) rename tests/{repackage/test_athena_prompt.py => unit/test_confirm_cost.py} (65%) rename tests/{repackage/test_duckdb_source.py => unit/test_duckdb.py} (62%) create mode 100644 tests/unit/test_sql_query.py diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index 27a2e11..6d76955 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -171,7 +171,7 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): import fsspec from cdx_toolkit.repackage.hf_utils import is_hf_url - from cdx_toolkit.repackage.sources.records import MissingWarcLocation, RecordsSource + from cdx_toolkit.repackage.records_source import MissingWarcLocation, RecordsSource try: source = RecordsSource(records, args.warc_download_prefix, sort=not args.no_sort) diff --git a/cdx_toolkit/repackage/sources/athena.py b/cdx_toolkit/indexes/athena.py similarity index 59% rename from cdx_toolkit/repackage/sources/athena.py rename to cdx_toolkit/indexes/athena.py index 50c1be1..27b5097 100644 --- a/cdx_toolkit/repackage/sources/athena.py +++ b/cdx_toolkit/indexes/athena.py @@ -1,16 +1,11 @@ -import itertools +''' +Running queries with AWS Athena (boto3). +''' import logging import time -from typing import Iterator, Iterable, Optional +from typing import Iterable, Iterator, Optional from cdx_toolkit.aws import botocore_config -from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.sources.base import RangeJobSource -from cdx_toolkit.repackage.sources.sql_base import ( - validate_result_columns, - join_warc_url, - REQUIRED_RESULT_COLUMNS, -) logger = logging.getLogger(__name__) @@ -19,51 +14,22 @@ ATHENA_USD_PER_TB = 5.0 -class AthenaSource(RangeJobSource): - """RangeJobs from a query against the CC columnar index via AWS Athena.""" - - def __init__( - self, - *, - query: str, - database: str, - s3_output_location: str, - warc_download_prefix: Optional[str], - region_name: Optional[str] = None, - max_wait_time: int = 300, - ): - self.query = query - self.database = database - self.s3_output_location = s3_output_location - self.warc_download_prefix = warc_download_prefix - self.region_name = region_name - self.max_wait_time = max_wait_time - - def _make_client(self): - import boto3 - - # retries: Common Crawl's recommended AWS settings, unless configured by the user (see cdx_toolkit.aws) - config = botocore_config(region_name=self.region_name, read_timeout=60) - return boto3.client('athena', config=config) - - def iter_rows(self) -> Iterator[dict]: - """Run the query and yield the result rows as dicts (column name -> string value).""" - client = self._make_client() - query_execution_id = run_athena_query( - client, self.query, self.database, self.s3_output_location, self.max_wait_time - ) - try: - yield from iter_result_rows(client, query_execution_id) - finally: - report_query_cost(client, query_execution_id) - - def iter_range_jobs(self) -> Iterator[RangeJob]: - rows = self.iter_rows() - first = next(rows, None) - if first is not None: - validate_result_columns(list(first)) - for row in itertools.chain([first], rows): - yield row_to_range_job(row, self.warc_download_prefix) +def make_client(region_name: Optional[str] = None): + import boto3 + + # retries: Common Crawl's recommended AWS settings, unless configured by the user (see cdx_toolkit.aws) + return boto3.client('athena', config=botocore_config(region_name=region_name, read_timeout=60)) + + +def iter_athena_rows(query: str, database: str, s3_output_location: str, region_name: Optional[str] = None, + max_wait_time: int = 300) -> Iterator[dict]: + """Run the query and yield the result rows as dicts (column name -> string value).""" + client = make_client(region_name) + query_execution_id = run_athena_query(client, query, database, s3_output_location, max_wait_time) + try: + yield from iter_result_rows(client, query_execution_id) + finally: + report_query_cost(client, query_execution_id) def run_athena_query(client, query: str, database: str, s3_output_location: str, max_wait_time: int = 300) -> str: @@ -163,26 +129,3 @@ def iter_result_rows(client, query_execution_id: str) -> Iterable[dict]: for row in rows: row_data = [cell.get('VarCharValue', None) for cell in row['Data']] yield dict(zip(column_names, row_data)) - - -def row_to_range_job(row: dict, warc_download_prefix: Optional[str]) -> RangeJob: - warc_filename = row['warc_filename'] - extra = {k: v for k, v in row.items() if k not in REQUIRED_RESULT_COLUMNS} - return RangeJob( - url=join_warc_url(warc_download_prefix, warc_filename), - offset=int(row['warc_record_offset']), - length=int(row['warc_record_length']), - filename=warc_filename, - extra=extra or None, - ) - - -def iter_range_jobs(client, query_execution_id: str, warc_download_prefix: Optional[str]) -> Iterable[RangeJob]: - """Retrieve query results and convert each row to a RangeJob""" - rows = iter_result_rows(client, query_execution_id) - first = next(rows, None) - if first is None: - return - validate_result_columns(list(first)) - for row in itertools.chain([first], rows): - yield row_to_range_job(row, warc_download_prefix) diff --git a/cdx_toolkit/indexes/duckdb.py b/cdx_toolkit/indexes/duckdb.py new file mode 100644 index 0000000..65bbf5a --- /dev/null +++ b/cdx_toolkit/indexes/duckdb.py @@ -0,0 +1,80 @@ +''' +Running queries with DuckDB, e.g. over the columnar URL index on S3 or on Hugging Face. +''' +import logging +from typing import Iterator, List, Optional + +from cdx_toolkit.aws import aws_region, duckdb_settings + + +logger = logging.getLogger(__name__) + +try: + import duckdb + _HAS_DUCKDB = True +except ImportError: # pragma: no cover - exercised in minimal installs + duckdb = None + _HAS_DUCKDB = False + + +def build_from_clause(index_path: str, crawls: Optional[List[str]]) -> str: + """Build a read_parquet(...) FROM expression over the CC columnar index. + + When crawls are given we glob only those crawl partitions (the pruning lever); + otherwise we glob every crawl (expensive -> the cost guard fires).""" + base = index_path.rstrip('/') + if crawls: + globs = [f"'{base}/crawl={c}/subset=warc/*.parquet'" for c in crawls] + else: + globs = [f"'{base}/crawl=*/subset=warc/*.parquet'"] + return f"read_parquet([{', '.join(globs)}], hive_partitioning=true)" + + +def iter_duckdb_rows(query: str, region_name: Optional[str] = None) -> Iterator[dict]: + """Run the query and yield the result rows as dicts (column name -> value). + + AWS region/credentials come from the environment (the AWS credential chain); the + public bucket is readable with valid credentials.""" + if not _HAS_DUCKDB: + raise RuntimeError( + 'DuckDB engine requires optional dependencies. Install cdx_toolkit[duckdb].' + ) + + logger.info('Executing DuckDB query: %s', query) + + con = duckdb.connect() + try: + con.execute('INSTALL httpfs; LOAD httpfs;') + # CC timestamps are UTC (timestamp literals in queries, timestamps in the results) + con.execute("SET TimeZone='UTC';") + region = aws_region(region_name) + try: + # find AWS credentials like boto3 does (env, ~/.aws profiles, SSO, instance roles) + con.execute('INSTALL aws; LOAD aws;') + con.execute(f"CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION '{region}');") + except Exception as e: # pragma: no cover - no credentials or aws extension unavailable + logger.debug('duckdb AWS credential chain not used: %r', e) + # region, retries and URL style: Common Crawl's recommended S3 settings, unless configured by + # the user (see cdx_toolkit.aws); a timeout of 2 minutes for large parquet partitions + for stmt in duckdb_settings(region_name) + ['SET http_timeout=120;']: + try: + con.execute(stmt) + except Exception as e: # pragma: no cover - setting unsupported on this duckdb + logger.debug('duckdb setting skipped: %s (%r)', stmt, e) + if 'hf://buckets/' in query: + # DuckDB cannot read Hugging Face Storage Buckets itself; use their fsspec filesystem. + from huggingface_hub import HfFileSystem + + con.register_filesystem(HfFileSystem()) + cur = con.execute(query) + + col_names = [d[0] for d in cur.description] + + while True: + rows = cur.fetchmany(1000) + if not rows: + break + for row in rows: + yield dict(zip(col_names, row)) + finally: + con.close() diff --git a/cdx_toolkit/indexes/sql.py b/cdx_toolkit/indexes/sql.py index b6ebd4b..1cc5267 100644 --- a/cdx_toolkit/indexes/sql.py +++ b/cdx_toolkit/indexes/sql.py @@ -8,13 +8,12 @@ from typing import Iterator, List, Mapping, Optional, Tuple from urllib.parse import parse_qs, urlparse -from cdx_toolkit.repackage.sources.base import CostEstimate from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up -from cdx_toolkit.repackage.sources.sql_base import build_sql, resolve_crawl_names from .base import IndexBackend, UsageError from .patterns import url_pattern_to_surts from .query import IndexQuery, field_flag +from .sql_query import build_sql, resolve_crawl_names LOGGER = logging.getLogger(__name__) @@ -45,20 +44,18 @@ GUIDED_OPTIONS = ('urls', 'hostnames', 'domains', 'crawl', 'from_ts', 'to', 'languages', 'limit') -def confirm_cost(estimate: Optional[CostEstimate], confirmed: bool) -> None: +def confirm_cost(n_crawls: Optional[int], engine: str, confirmed: bool) -> None: """Prompt before running a potentially expensive index scan. - A scan bounded to <= LARGE_CRAWL_SET_THRESHOLD crawls runs without a prompt. - Otherwise (no crawl filter, a large crawl set, or unverifiable raw SQL) we confirm - interactively, abort in non-interactive sessions, unless `confirmed` (--yes).""" - if confirmed or estimate is None: + `n_crawls` is the number of crawls the scan is restricted to, None if it is not restricted (or it cannot be + verified, e.g. for raw SQL). A scan bounded to <= LARGE_CRAWL_SET_THRESHOLD crawls runs without a prompt. + Otherwise we confirm interactively, abort in non-interactive sessions, unless `confirmed` (--yes).""" + if confirmed: return - n_crawls = estimate.n_crawls if n_crawls is not None and n_crawls <= LARGE_CRAWL_SET_THRESHOLD: return - engine = estimate.engine if n_crawls is None: reason = (f'This {engine} query is not restricted to specific crawls (or uses custom SQL ' 'whose crawl partition pruning could not be verified) and may scan ALL crawls.') @@ -164,7 +161,6 @@ class SqlBackend(IndexBackend): def __init__(self, location: str, assume_yes: bool = False): self.location = location self.assume_yes = assume_yes - self._prepared = None url_file_hint = '' @@ -215,13 +211,13 @@ def prepare(self, query: IndexQuery, order_by: bool = True) -> Tuple[str, Option def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: sql, n_crawls = self.prepare(query) - confirm_cost(CostEstimate(n_crawls=n_crawls, engine=self.engine), self.assume_yes) + confirm_cost(n_crawls, self.engine, self.assume_yes) for row in self.iter_sql_rows(sql): yield normalize_row(row) def count(self, query: IndexQuery) -> int: sql, n_crawls = self.prepare(query, order_by=False) - confirm_cost(CostEstimate(n_crawls=n_crawls, engine=self.engine), self.assume_yes) + confirm_cost(n_crawls, self.engine, self.assume_yes) row = next(self.iter_sql_rows(f'SELECT count(*) AS n FROM ({sql}) AS q')) return int(row['n']) @@ -250,18 +246,17 @@ def __init__(self, location: str, assume_yes: bool = False): super().__init__(location, assume_yes) def from_clause(self, crawls): - from cdx_toolkit.repackage.sources.duckdb import _build_from_clause + from .duckdb import build_from_clause if self.location.endswith('.parquet'): # file(s), possibly a glob return f"read_parquet('{self.location}', hive_partitioning=true)", False # hive-partitioned index directory: prune the crawls in the glob - return _build_from_clause(self.location, crawls), True + return build_from_clause(self.location, crawls), True def iter_sql_rows(self, sql): - from cdx_toolkit.repackage.sources.duckdb import DuckDbSource + from .duckdb import iter_duckdb_rows - source = DuckDbSource(query=sql) - return source.iter_rows() + return iter_duckdb_rows(sql) class AthenaBackend(SqlBackend): @@ -288,16 +283,9 @@ def from_clause(self, crawls): return self.table, False def iter_sql_rows(self, sql): - from cdx_toolkit.repackage.sources.athena import AthenaSource - - source = AthenaSource( - query=sql, - database=self.database, - s3_output_location=self.s3_output_location, - warc_download_prefix=None, - region_name=self.region, - ) - return source.iter_rows() + from .athena import iter_athena_rows + + return iter_athena_rows(sql, self.database, self.s3_output_location, region_name=self.region) def parse_athena_location(location: str) -> Tuple[str, str, str, str]: diff --git a/cdx_toolkit/repackage/sources/sql_base.py b/cdx_toolkit/indexes/sql_query.py similarity index 79% rename from cdx_toolkit/repackage/sources/sql_base.py rename to cdx_toolkit/indexes/sql_query.py index 7e1cb88..29f8acd 100644 --- a/cdx_toolkit/repackage/sources/sql_base.py +++ b/cdx_toolkit/indexes/sql_query.py @@ -1,14 +1,16 @@ +''' +SQL queries for the columnar URL index (Athena, DuckDB): escaping, the WHERE clause of the guided queries, crawls. +''' import logging import re from typing import List, Optional from cdx_toolkit.commoncrawl import normalize_crawl, get_cc_endpoints, match_cc_crawls +from .patterns import domain_to_surts -logger = logging.getLogger(__name__) -# Required output columns that any index query (built or raw) must provide. -REQUIRED_RESULT_COLUMNS = ('warc_filename', 'warc_record_offset', 'warc_record_length') +logger = logging.getLogger(__name__) # Hostnames, TLDs and crawl names only ever contain these characters. Validating # against this set both prevents SQL injection and catches malformed input early. @@ -57,8 +59,6 @@ def domain_predicate(domain: str) -> str: The SURT prefixes give the exact host boundary; the registered domain, which is one of the suffixes of the domain, makes the query cheaper (an equality the engines can push down).""" - from cdx_toolkit.indexes.patterns import domain_to_surts - surts = domain_to_surts(domain) labels = surts[0][:-1].split(',')[::-1] # canonical, e.g. without www. suffixes = ['.'.join(labels[i:]) for i in range(len(labels) - 1)] or labels @@ -139,14 +139,14 @@ def build_sql( `from_clause` is the text following FROM (e.g. `ccindex` for Athena, or a `read_parquet(...)` expression for DuckDB). When `order_by` is set, results are sorted by (warc_filename, warc_record_offset) for fetch-time read locality. - `columns` defaults to the REQUIRED_RESULT_COLUMNS.""" + `columns` defaults to the WARC location columns.""" where_sql = build_where_sql( url_host_names, crawls, url_host_registered_domains=url_host_registered_domains, conditions=conditions, url_surtkey_prefixes=url_surtkey_prefixes, url_surtkeys=url_surtkeys, ) order_sql = f'\n {ORDER_BY_SQL}' if order_by else '' limit_sql = f'\n LIMIT {limit}' if limit and limit > 0 else '' - columns_sql = ', '.join(columns or REQUIRED_RESULT_COLUMNS) + columns_sql = ', '.join(columns or ('warc_filename', 'warc_record_offset', 'warc_record_length')) return f""" SELECT @@ -155,47 +155,6 @@ def build_sql( WHERE {where_sql}{order_sql}{limit_sql}""" -def build_athena_query( - url_host_names: Optional[List[str]] = None, - crawls: Optional[List[str]] = None, - limit: int = 0, - table: str = 'ccindex', - url_host_registered_domains: Optional[List[str]] = None, - order_by: bool = True, -) -> str: - """Athena flavour of build_sql (FROM
).""" - return build_sql( - table, url_host_names, crawls=crawls, limit=limit, - url_host_registered_domains=url_host_registered_domains, - order_by=order_by, - ) - - -def validate_result_columns(column_names) -> None: - """Raise a clear error if the query result lacks the required columns.""" - missing = [c for c in REQUIRED_RESULT_COLUMNS if c not in (column_names or [])] - if missing: - raise ValueError( - 'Index query result is missing required columns: ' + ', '.join(missing) + - '. The query (including a raw --query) must SELECT ' + - ', '.join(REQUIRED_RESULT_COLUMNS) + '.' - ) - - -def join_warc_url(prefix: Optional[str], warc_filename: str) -> str: - """Join a download prefix with a warc_filename robustly. - - - if warc_filename is already absolute (contains '://'), return it unchanged - (supports custom queries / self-contained CSVs whose value is a full URL); - - if prefix is empty/None, return warc_filename unchanged; - - otherwise join with exactly one '/' (no double slash, no missing slash).""" - if '://' in warc_filename: - return warc_filename - if not prefix: - return warc_filename - return prefix.rstrip('/') + '/' + warc_filename.lstrip('/') - - def endpoint_to_crawl_name(endpoint: str) -> str: """Turn a collinfo cdx-api endpoint into its crawl name. diff --git a/cdx_toolkit/repackage/data_classes.py b/cdx_toolkit/repackage/data_classes.py index 3e897cd..16ea3a0 100644 --- a/cdx_toolkit/repackage/data_classes.py +++ b/cdx_toolkit/repackage/data_classes.py @@ -1,6 +1,6 @@ import asyncio import time -from dataclasses import dataclass, field +from dataclasses import dataclass from cdx_toolkit.repackage.s3_utils import is_s3_url, parse_s3_uri, with_retries from cdx_toolkit.repackage.hf_utils import is_hf_url @@ -64,14 +64,9 @@ class RangeJob: offset: int length: int records_count: int = 1 - # Relative WARC filename (e.g. crawl-data/...warc.gz) as known by the source, - # used when materializing a non-self-contained range-jobs CSV. `url` stays - # authoritative for fetching. + # WARC filename of the record as given in the index (e.g. crawl-data/...warc.gz, relative to the download + # prefix), used to sort and shard the jobs by WARC file; `url` is authoritative for fetching. filename: Optional[str] = None - # Extra columns from a raw SQL --query (e.g. content_languages), kept only for - # CSV materialization / analysis. Never used for fetching. Excluded from - # equality/hash so RangeJob stays hashable despite the dict value. - extra: Optional[dict] = field(default=None, compare=False) def is_s3(self): return is_s3_url(self.url) diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index d5ecfa6..f0836dd 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -5,8 +5,8 @@ all cores needs multiple processes. This orchestrator keeps that entirely inside ``cdxt repackage`` (no external driver): -1. Drain the configured source once and split it into N self-contained shard CSVs, - hashed by WARC filename so every record of a file stays in one shard. +1. Drain the configured source once and split it into N shard files (JSONL records with the full + WARC URL), hashed by WARC filename so every record of a file stays in one shard. 2. Run one worker process per (non-empty) shard concurrently. Each worker **rotates its output at ``--size``**; every output file is a self-contained WARC with its own ``warcinfo`` record. @@ -36,8 +36,9 @@ from typing import Dict, List, Optional, Tuple from cdx_toolkit.aws import configure_fsspec_s3 +from cdx_toolkit.record_io import RecordWriter, read_records from cdx_toolkit.repackage.data_classes import RepackageResult -from cdx_toolkit.repackage.sources.csv import CsvSource, RangeJobCsvWriter +from cdx_toolkit.repackage.records_source import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter from cdx_toolkit.repackage.warc_utils import generate_warc_filename from cdx_toolkit.repackage.s3_utils import is_s3_url, parse_s3_uri @@ -78,7 +79,7 @@ def _run_worker(cfg: Dict) -> RepackageResult: """Worker entrypoint (runs in a child process): fetch one shard into the output WARCs.""" if cfg.get('uvloop'): os.environ['CDXT_UVLOOP'] = '1' - source = CsvSource(cfg['shard_csv'], warc_download_prefix=None, sort=True) + source = RecordsSource(read_records(cfg['shard'], 'jsonl'), warc_download_prefix=None, sort=True) wf = WARCFilter( source=source, prefix_path=cfg['prefix_path'], @@ -96,23 +97,25 @@ def _run_worker(cfg: Dict) -> RepackageResult: def _split_source_to_shards(source, n: int, out_dir: str) -> Tuple[List[str], List[int]]: - """Drain the source once into N self-contained shard CSVs, hashed by WARC file. + """Drain the source once into N shard files, hashed by WARC file; the records have the full WARC URL as + filename, so they are read without a download prefix. Returns the shard paths and the number of range jobs in each shard.""" - paths = [os.path.join(out_dir, f'shard.{i}.csv') for i in range(n)] - writers = [RangeJobCsvWriter(p) for p in paths] + paths = [os.path.join(out_dir, f'shard.{i}.jsonl') for i in range(n)] + files = [open(p, 'w', encoding='utf-8') for p in paths] + writers = [RecordWriter(f, 'jsonl') for f in files] counts = [0] * n total = 0 try: for job in source.iter_range_jobs(): key = (job.filename or job.url or '').encode() idx = int(hashlib.md5(key).hexdigest(), 16) % n - writers[idx].write(job) + writers[idx].write({'filename': job.url, 'offset': job.offset, 'length': job.length}) counts[idx] += 1 total += 1 finally: - for w in writers: - w.close() + for f in files: + f.close() logger.info('Sharded %d range jobs into %d shards: %s', total, n, counts) return paths, counts @@ -137,9 +140,9 @@ def run_multiprocess_repackage( tmp_dir = tempfile.mkdtemp(prefix='cdxt_shards_') start = time.time() try: - shard_csvs, counts = _split_source_to_shards(source, n_processes, tmp_dir) + shards, counts = _split_source_to_shards(source, n_processes, tmp_dir) # empty shards would only produce WARC files without records - shard_csvs = [path for path, count in zip(shard_csvs, counts) if count > 0] or shard_csvs[:1] + shards = [path for path, count in zip(shards, counts) if count > 0] or shards[:1] # The WARC writer supports S3 and local FS only. Other fsspec destinations # (e.g. an hf:// bucket) are staged in a local dir and copied afterwards. @@ -152,7 +155,7 @@ def run_multiprocess_repackage( configs = [ dict( - shard_csv=shard_csv, + shard=shard, prefix_path=write_prefix, subprefix=writer_subprefix, writer_info=writer_info, @@ -164,7 +167,7 @@ def run_multiprocess_repackage( max_file_size=max_file_size, uvloop=uvloop, ) - for shard_csv in shard_csvs + for shard in shards ] logger.info('Launching %d worker processes x %d readers', len(configs), readers_per_process) diff --git a/cdx_toolkit/repackage/sources/records.py b/cdx_toolkit/repackage/records_source.py similarity index 66% rename from cdx_toolkit/repackage/sources/records.py rename to cdx_toolkit/repackage/records_source.py index 518eaef..8dbe821 100644 --- a/cdx_toolkit/repackage/sources/records.py +++ b/cdx_toolkit/repackage/records_source.py @@ -1,9 +1,10 @@ +''' +The captures to repackage: index records (as written by `cdxt index`) -> range jobs for the WARC fetcher. +''' import logging from typing import Iterable, Iterator, Mapping, Optional from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.sources.base import RangeJobSource -from cdx_toolkit.repackage.sources.sql_base import join_warc_url logger = logging.getLogger(__name__) @@ -15,7 +16,21 @@ class MissingWarcLocation(ValueError): """A record has no WARC location (filename, offset, length), e.g. an Internet Archive capture.""" -class RecordsSource(RangeJobSource): +def join_warc_url(prefix: Optional[str], warc_filename: str) -> str: + """Join a download prefix with a warc_filename robustly. + + - if warc_filename is already absolute (contains '://'), return it unchanged + (supports custom queries / self-contained index files whose value is a full URL); + - if prefix is empty/None, return warc_filename unchanged; + - otherwise join with exactly one '/' (no double slash, no missing slash).""" + if '://' in warc_filename: + return warc_filename + if not prefix: + return warc_filename + return prefix.rstrip('/') + '/' + warc_filename.lstrip('/') + + +class RecordsSource: """RangeJobs from index records (as written by `cdxt index`) with filename, offset and length fields. With `sort` (the default), jobs are sorted by (WARC file, record offset) for better @@ -29,6 +44,7 @@ def __init__(self, records: Iterable[Mapping], warc_download_prefix: Optional[st self.jobs = sorted(self.jobs, key=lambda j: (j.filename or j.url, j.offset)) def iter_range_jobs(self) -> Iterator[RangeJob]: + """Yield the range jobs; `url` is authoritative for fetching, `filename` is the filename of the record.""" yield from self.jobs def to_range_job(self, record: Mapping) -> RangeJob: diff --git a/cdx_toolkit/repackage/sources/__init__.py b/cdx_toolkit/repackage/sources/__init__.py deleted file mode 100644 index af3d689..0000000 --- a/cdx_toolkit/repackage/sources/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from cdx_toolkit.repackage.sources.base import RangeJobSource, CostEstimate - -__all__ = ['RangeJobSource', 'CostEstimate'] diff --git a/cdx_toolkit/repackage/sources/base.py b/cdx_toolkit/repackage/sources/base.py deleted file mode 100644 index faa4e4b..0000000 --- a/cdx_toolkit/repackage/sources/base.py +++ /dev/null @@ -1,30 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Iterator, NamedTuple, Optional - -from cdx_toolkit.repackage.data_classes import RangeJob - - -class CostEstimate(NamedTuple): - """Describes the scan a source is about to run, for the cost-confirmation guard. - - n_crawls is the number of crawls the scan is bounded to, or None when the scan - is unbounded / its pruning cannot be verified (e.g. raw SQL, all-crawls glob).""" - - n_crawls: Optional[int] - engine: str - - -class RangeJobSource(ABC): - """A source of RangeJobs for the repackage pipeline. - - A source owns its own stage-1 resource (Athena client, DuckDB connection, or an - fsspec file handle) and yields RangeJobs synchronously; the pipeline orchestrator - bridges the sync generator into the async fetch/write stages, and owns queueing, - counting, and stop-sentinel emission.""" - - @abstractmethod - def iter_range_jobs(self) -> Iterator[RangeJob]: - """Yield RangeJobs. Implementations open their own client/connection lazily - and close it in a finally. Each RangeJob carries `url` (authoritative for - fetching) and, where known, the relative `filename`.""" - raise NotImplementedError # pragma: no cover diff --git a/cdx_toolkit/repackage/sources/csv.py b/cdx_toolkit/repackage/sources/csv.py deleted file mode 100644 index 3c4f270..0000000 --- a/cdx_toolkit/repackage/sources/csv.py +++ /dev/null @@ -1,120 +0,0 @@ -import csv -import logging -from typing import Iterator, Optional - -import fsspec - -from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.sources.base import RangeJobSource -from cdx_toolkit.repackage.sources.sql_base import join_warc_url - - -logger = logging.getLogger(__name__) - -# Fetch-job columns name the WARC *location* (mirroring the columnar index's column -# names) so they never collide with extra analysis columns a raw --query may SELECT -# -- e.g. the index's own `url` (the captured page URL). `warc_filename` is a filename -# relative to the WARC download prefix, or a full URL (s3://, https://, hf://...). -FILENAME_FIELDS = ['warc_filename', 'warc_record_offset', 'warc_record_length'] - -# Columns the reader interprets as the fetch job; everything else round-trips as extra. -_KNOWN_FIELDS = set(FILENAME_FIELDS) - - -class RangeJobCsvWriter: - """Write RangeJobs to a CSV (local or remote via fsspec). - - The `warc_filename` column holds the full download URL of each job, so the CSV can - be read without knowing the WARC download prefix. Any extra columns carried on a - RangeJob (from a raw SQL --query) are appended after the fetch-job columns. The - header is written lazily from the first job so its extra columns can be discovered.""" - - def __init__(self, path: str): - self.path = path - self._ctx = fsspec.open(path, 'wt', newline='') - self._fh = self._ctx.__enter__() - self._writer = None - - def _init_writer(self, fieldnames) -> None: - self._writer = csv.DictWriter( - self._fh, fieldnames=fieldnames, extrasaction='ignore', restval='' - ) - self._writer.writeheader() - - def write(self, job: RangeJob) -> None: - row = { - 'warc_filename': job.url, - 'warc_record_offset': job.offset, - 'warc_record_length': job.length, - } - if job.extra: - row.update(job.extra) - - if self._writer is None: - extra_fields = [k for k in (job.extra or {}) if k not in FILENAME_FIELDS] - self._init_writer(FILENAME_FIELDS + extra_fields) - self._writer.writerow(row) - - def close(self) -> None: - if self._ctx is not None: - # No rows written: still emit the base header for a valid (empty) file. - if self._writer is None: - self._init_writer(FILENAME_FIELDS) - self._ctx.__exit__(None, None, None) - self._ctx = None - self._fh = None - - -class CsvSource(RangeJobSource): - """RangeJobs read from a CSV/TSV with `warc_filename`, `warc_record_offset` and - `warc_record_length` columns. - - A `warc_filename` that is a full URL (s3://, https://, hf://...) is used as-is, - otherwise the WARC download prefix is prepended. Any other columns round-trip onto - RangeJob.extra. TSV is detected from a `.tsv`/`.tsv.gz` extension; `.gz` inputs are - decompressed. - - With `sort` (the default), jobs are sorted by (WARC file, record offset) before - being emitted -- grouping same-file records with ascending offsets for better S3 - range-read locality. This buffers all rows in memory; pass sort=False to stream - an already-sorted file without buffering.""" - - def __init__(self, path: str, warc_download_prefix: Optional[str], sort: bool = True): - self.path = path - self.warc_download_prefix = warc_download_prefix - self.sort = sort - - def iter_range_jobs(self) -> Iterator[RangeJob]: - path = str(self.path) - delimiter = '\t' if path.endswith(('.tsv', '.tsv.gz')) else ',' - compression = 'gzip' if path.endswith('.gz') else None - - buffered = [] if self.sort else None - with fsspec.open(self.path, 'rt', newline='', compression=compression) as fh: - reader = csv.DictReader(fh, delimiter=delimiter) - missing = [f for f in FILENAME_FIELDS if f not in (reader.fieldnames or [])] - if missing: - raise ValueError( - f'range-jobs CSV {self.path} must have the columns {", ".join(FILENAME_FIELDS)} ' - f'(got header: {reader.fieldnames})' - ) - - for row in reader: - filename = row['warc_filename'] - extra = {k: v for k, v in row.items() if k not in _KNOWN_FIELDS} - job = RangeJob( - url=join_warc_url(self.warc_download_prefix, filename), - offset=int(row['warc_record_offset']), - length=int(row['warc_record_length']), - filename=filename, - extra=extra or None, - ) - if buffered is None: - yield job - else: - buffered.append(job) - - if buffered is not None: - buffered.sort(key=lambda j: (j.filename or j.url or '', j.offset)) - for job in buffered: - yield job diff --git a/cdx_toolkit/repackage/sources/duckdb.py b/cdx_toolkit/repackage/sources/duckdb.py deleted file mode 100644 index 065fc8f..0000000 --- a/cdx_toolkit/repackage/sources/duckdb.py +++ /dev/null @@ -1,107 +0,0 @@ -import logging -from typing import Iterator, List, Optional - -from cdx_toolkit.aws import aws_region, duckdb_settings -from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.sources.base import RangeJobSource -from cdx_toolkit.repackage.sources.sql_base import ( - validate_result_columns, - join_warc_url, - REQUIRED_RESULT_COLUMNS, -) - - -logger = logging.getLogger(__name__) - -try: - import duckdb - _HAS_DUCKDB = True -except ImportError: # pragma: no cover - exercised in minimal installs - duckdb = None - _HAS_DUCKDB = False - - -def _build_from_clause(index_path: str, crawls: Optional[List[str]]) -> str: - """Build a read_parquet(...) FROM expression over the CC columnar index. - - When crawls are given we glob only those crawl partitions (the pruning lever); - otherwise we glob every crawl (expensive -> the cost guard fires).""" - base = index_path.rstrip('/') - if crawls: - globs = [f"'{base}/crawl={c}/subset=warc/*.parquet'" for c in crawls] - else: - globs = [f"'{base}/crawl=*/subset=warc/*.parquet'"] - return f"read_parquet([{', '.join(globs)}], hive_partitioning=true)" - - -class DuckDbSource(RangeJobSource): - """RangeJobs (or rows) from a SQL query run with DuckDB, e.g. over the CC columnar index on S3. - - AWS region/credentials come from the environment (the AWS credential chain); the - public bucket is readable with valid credentials.""" - - def __init__(self, *, query: str, warc_download_prefix: Optional[str] = None, region_name: Optional[str] = None): - self.query = query - self.warc_download_prefix = warc_download_prefix - self.region_name = region_name - - def iter_rows(self, validate: bool = False) -> Iterator[dict]: - """Run the query and yield the result rows as dicts (column name -> value).""" - if not _HAS_DUCKDB: - raise RuntimeError( - 'DuckDB engine requires optional dependencies. Install cdx_toolkit[duckdb].' - ) - - query = self.query - logger.info('Executing DuckDB query: %s', query) - - con = duckdb.connect() - try: - con.execute('INSTALL httpfs; LOAD httpfs;') - # CC timestamps are UTC (timestamp literals in queries, timestamps in the results) - con.execute("SET TimeZone='UTC';") - region = aws_region(self.region_name) - try: - # find AWS credentials like boto3 does (env, ~/.aws profiles, SSO, instance roles) - con.execute('INSTALL aws; LOAD aws;') - con.execute(f"CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION '{region}');") - except Exception as e: # pragma: no cover - no credentials or aws extension unavailable - logger.debug('duckdb AWS credential chain not used: %r', e) - # region, retries and URL style: Common Crawl's recommended S3 settings, unless configured by - # the user (see cdx_toolkit.aws); a timeout of 2 minutes for large parquet partitions - for stmt in duckdb_settings(self.region_name) + ['SET http_timeout=120;']: - try: - con.execute(stmt) - except Exception as e: # pragma: no cover - setting unsupported on this duckdb - logger.debug('duckdb setting skipped: %s (%r)', stmt, e) - if 'hf://buckets/' in query: - # DuckDB cannot read Hugging Face Storage Buckets itself; use their fsspec filesystem. - from huggingface_hub import HfFileSystem - - con.register_filesystem(HfFileSystem()) - cur = con.execute(query) - - col_names = [d[0] for d in cur.description] - if validate: - validate_result_columns(col_names) - - while True: - rows = cur.fetchmany(1000) - if not rows: - break - for row in rows: - yield dict(zip(col_names, row)) - finally: - con.close() - - def iter_range_jobs(self) -> Iterator[RangeJob]: - for row in self.iter_rows(validate=True): - warc_filename = row['warc_filename'] - extra = {name: value for name, value in row.items() if name not in REQUIRED_RESULT_COLUMNS} - yield RangeJob( - url=join_warc_url(self.warc_download_prefix, warc_filename), - offset=int(row['warc_record_offset']), - length=int(row['warc_record_length']), - filename=warc_filename, - extra=extra or None, - ) diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py index d7ce322..7c2f044 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/warc_filter.py @@ -15,7 +15,6 @@ from cdx_toolkit.repackage.hf_utils import make_hf_reader from cdx_toolkit.repackage.data_classes import RangeJob, RangePayload, RepackageResult, ThroughputTracker from cdx_toolkit.repackage.warc_utils import create_new_writer_with_header -from cdx_toolkit.repackage.sources.base import RangeJobSource from cdx_toolkit.repackage.warc_utils import get_bytes_from_warc_record, get_metadata_record_from_path @@ -72,7 +71,7 @@ def __init__( self, prefix_path: str, writer_info: Dict, - source: RangeJobSource, + source, writer_subprefix: Optional[str] = None, write_paths_as_metadata_records: Optional[List[str]] = None, log_every_n: int = 1000, @@ -92,7 +91,7 @@ def __init__( """Initialize the WARC filter. Args: - source: RangeJobSource that yields the WARC ranges to repackage. + source: The WARC ranges to repackage: an object with iter_range_jobs() (e.g. a RecordsSource). prefix_path: Output path prefix for filtered WARC files. writer_info: Dictionary containing writer metadata. writer_subprefix: Optional subprefix for writer output paths. diff --git a/docs/notes/warc-fetcher-performance.md b/docs/notes/warc-fetcher-performance.md index 2bd1c01..bb64d86 100644 --- a/docs/notes/warc-fetcher-performance.md +++ b/docs/notes/warc-fetcher-performance.md @@ -19,7 +19,7 @@ Status legend: **[done]** implemented on this branch · **[proposed]** not yet i Three async stages on a single `asyncio` event loop (`cdx_toolkit/repackage/warc_filter.py`): -1. **Produce** — the selected `RangeJobSource` yields `RangeJob`s (one per WARC record: +1. **Produce** — the source (`RecordsSource`, the captures of the index file) yields `RangeJob`s (one per WARC record: url + byte offset + length). The source is a *sync* generator run in a worker thread (`asyncio.to_thread(drain)`, `warc_filter.py:~276`), pushing each job onto a bounded `range_jobs_queue` via `run_coroutine_threadsafe(...).result()`. diff --git a/tests/repackage/test_athena_query_builder.py b/tests/repackage/test_athena_query_builder.py deleted file mode 100644 index 45878e0..0000000 --- a/tests/repackage/test_athena_query_builder.py +++ /dev/null @@ -1,223 +0,0 @@ -import pytest - -from cdx_toolkit.repackage.sources.sql_base import ( - build_athena_query, - escape_sql_literal, - validate_result_columns, - join_warc_url, -) -from cdx_toolkit.repackage.sources.athena import run_athena_query, iter_range_jobs - - -class _FakeAthenaClient: - """Minimal stand-in for a boto3 Athena client (no AWS, no network).""" - - def __init__(self, raise_on_wait=None): - self._raise_on_wait = raise_on_wait - self.started = [] - self.stopped = [] - - def start_query_execution(self, **kwargs): - self.started.append(kwargs) - return {'QueryExecutionId': 'qid-123'} - - def get_query_execution(self, QueryExecutionId): - if self._raise_on_wait is not None: - raise self._raise_on_wait - return {'QueryExecution': {'Status': {'State': 'SUCCEEDED'}, - 'Statistics': {'DataScannedInBytes': 1000}}} - - def stop_query_execution(self, QueryExecutionId): - self.stopped.append(QueryExecutionId) - - -def test_build_query_hostnames(): - q = build_athena_query(['example.com', 'test.org']) - assert "subset = 'warc'" in q - assert "url_host_tld = 'com'" in q - assert "url_host_tld = 'org'" in q - assert "url_host_name = 'example.com'" in q - assert "url_host_name = 'test.org'" in q - assert 'warc_filename, warc_record_offset, warc_record_length' in q - assert 'crawl IN' not in q - assert 'LIMIT' not in q - - -def test_build_query_domains_only(): - q = build_athena_query(url_host_registered_domains=['example.com']) - assert "url_host_registered_domain IN ('example.com')" in q - assert "url_host_tld = 'com'" in q - assert 'url_host_name' not in q - - -def test_build_query_hostnames_and_domains(): - q = build_athena_query(['www.example.com'], url_host_registered_domains=['example.org']) - assert "url_host_name = 'www.example.com'" in q - assert "url_host_registered_domain IN ('example.org')" in q - assert "url_host_tld = 'com'" in q - assert "url_host_tld = 'org'" in q - - -def test_build_query_requires_host_or_domain(): - with pytest.raises(ValueError): - build_athena_query() - - -def test_build_query_with_crawls(): - q = build_athena_query(['example.com'], crawls=['CC-MAIN-2025-33', 'CC-MAIN-2025-30']) - assert "crawl IN ('CC-MAIN-2025-33', 'CC-MAIN-2025-30')" in q - - -def test_build_query_limit(): - assert 'LIMIT 10' in build_athena_query(['example.com'], limit=10) - assert 'LIMIT' not in build_athena_query(['example.com'], limit=0) - - -def test_build_query_orders_by_default(): - q = build_athena_query(['example.com']) - assert 'ORDER BY warc_filename, warc_record_offset' in q - - -def test_build_query_order_by_false(): - assert 'ORDER BY' not in build_athena_query(['example.com'], order_by=False) - - -def test_build_query_order_by_precedes_limit(): - q = build_athena_query(['example.com'], limit=10) - assert q.index('ORDER BY') < q.index('LIMIT') - - -def test_build_query_requires_hostnames(): - with pytest.raises(ValueError): - build_athena_query([]) - - -def test_escape_sql_literal_valid(): - assert escape_sql_literal('example.com') == "'example.com'" - assert escape_sql_literal('CC-MAIN-2025-33') == "'CC-MAIN-2025-33'" - - -@pytest.mark.parametrize('bad', ["a';DROP TABLE x;--", "a' OR '1'='1", 'a b', "a'", "', '"]) -def test_escape_sql_literal_rejects_injection(bad): - with pytest.raises(ValueError): - escape_sql_literal(bad) - - -def test_build_query_rejects_injection_hostname(): - with pytest.raises(ValueError): - build_athena_query(["example.com'; DROP TABLE ccindex; --"]) - - -def test_validate_result_columns_ok(): - validate_result_columns(['warc_filename', 'warc_record_offset', 'warc_record_length', 'extra']) - - -def test_validate_result_columns_missing(): - with pytest.raises(ValueError) as e: - validate_result_columns(['warc_filename']) - assert 'warc_record_offset' in str(e.value) - assert 'warc_record_length' in str(e.value) - - -def test_join_warc_url_relative(): - assert ( - join_warc_url('https://data.commoncrawl.org', 'crawl-data/x.warc.gz') - == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' - ) - - -def test_join_warc_url_no_double_slash(): - assert ( - join_warc_url('https://data.commoncrawl.org/', 'crawl-data/x.warc.gz') - == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' - ) - assert join_warc_url('s3://bucket', '/crawl-data/x.warc.gz') == 's3://bucket/crawl-data/x.warc.gz' - - -def test_join_warc_url_empty_prefix(): - assert join_warc_url('', 'crawl-data/x.warc.gz') == 'crawl-data/x.warc.gz' - assert join_warc_url(None, 'crawl-data/x.warc.gz') == 'crawl-data/x.warc.gz' - - -def test_join_warc_url_absolute_filename_passthrough(): - assert join_warc_url('https://data.commoncrawl.org', 's3://cc/x.warc.gz') == 's3://cc/x.warc.gz' - assert join_warc_url('', 'https://host/x.warc.gz') == 'https://host/x.warc.gz' - - -class _FakePaginator: - """Yields Athena get_query_results-style pages (first row is the header).""" - - def __init__(self, pages): - self._pages = pages - - def paginate(self, **kwargs): - return iter(self._pages) - - -class _FakeResultsClient: - def __init__(self, pages): - self._pages = pages - - def get_paginator(self, name): - assert name == 'get_query_results' - return _FakePaginator(self._pages) - - -def _athena_page(columns, *value_rows): - header = {'Data': [{'VarCharValue': c} for c in columns]} - data = [{'Data': [{'VarCharValue': v} for v in row]} for row in value_rows] - return {'ResultSet': {'Rows': [header] + data}} - - -def test_iter_range_jobs_carries_extra_columns(): - pages = [_athena_page( - ['warc_filename', 'warc_record_offset', 'warc_record_length', 'content_languages', 'url'], - ['crawl-data/x.warc.gz', '100', '200', 'eng', 'https://example.com/page'], - )] - client = _FakeResultsClient(pages) - jobs = list(iter_range_jobs(client, 'qid', 'https://data.commoncrawl.org')) - assert len(jobs) == 1 - job = jobs[0] - assert job.url == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' - assert job.offset == 100 and job.length == 200 - assert job.filename == 'crawl-data/x.warc.gz' - # extra carries only the non-required columns (page URL distinct from warc URL) - assert job.extra == {'content_languages': 'eng', 'url': 'https://example.com/page'} - - -def test_iter_range_jobs_no_extra_columns_is_none(): - pages = [_athena_page( - ['warc_filename', 'warc_record_offset', 'warc_record_length'], - ['crawl-data/x.warc.gz', '1', '2'], - )] - jobs = list(iter_range_jobs(_FakeResultsClient(pages), 'qid', 'https://data.commoncrawl.org')) - assert jobs[0].extra is None - - -def test_run_athena_query_logs_sql(caplog): - import logging - client = _FakeAthenaClient() - with caplog.at_level(logging.INFO): - run_athena_query(client, 'SELECT my_special_column FROM ccindex', 'ccindex', 's3://b/out/') - assert 'SELECT my_special_column FROM ccindex' in caplog.text - - -def test_run_athena_query_success_no_cancel(): - client = _FakeAthenaClient() - qid = run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') - assert qid == 'qid-123' - assert client.stopped == [] - - -def test_run_athena_query_cancels_on_interrupt(): - client = _FakeAthenaClient(raise_on_wait=KeyboardInterrupt()) - with pytest.raises(KeyboardInterrupt): - run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') - assert client.stopped == ['qid-123'] - - -def test_run_athena_query_cancels_on_timeout(): - client = _FakeAthenaClient(raise_on_wait=TimeoutError('too slow')) - with pytest.raises(TimeoutError): - run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') - assert client.stopped == ['qid-123'] diff --git a/tests/repackage/test_csv_source.py b/tests/repackage/test_csv_source.py deleted file mode 100644 index b60e112..0000000 --- a/tests/repackage/test_csv_source.py +++ /dev/null @@ -1,130 +0,0 @@ -import csv - -import pytest - -from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.sources.csv import RangeJobCsvWriter, CsvSource - - -def test_writer_writes_full_urls_and_reads_them_back(tmp_path): - path = str(tmp_path / 'ranges.csv') - w = RangeJobCsvWriter(path) - w.write(RangeJob(url='s3://commoncrawl/crawl-data/x.warc.gz', offset=5, length=7, filename='crawl-data/x.warc.gz')) - w.close() - - with open(path, newline='') as f: - rows = list(csv.DictReader(f)) - assert rows[0] == { - 'warc_filename': 's3://commoncrawl/crawl-data/x.warc.gz', - 'warc_record_offset': '5', - 'warc_record_length': '7', - } - - # a full URL is used as-is, whatever the prefix - jobs = list(CsvSource(path, 'https://ignored').iter_range_jobs()) - assert jobs[0].url == 's3://commoncrawl/crawl-data/x.warc.gz' - assert jobs[0].offset == 5 and jobs[0].length == 7 - - -def test_csv_source_relative_filename_gets_prefix(tmp_path): - path = str(tmp_path / 'ranges.csv') - with open(path, 'w') as f: - f.write('warc_filename,warc_record_offset,warc_record_length\n') - f.write('crawl-data/x.warc.gz,10,20\n') - f.write('https://example.org/y.warc.gz,1,2\n') - jobs = list(CsvSource(path, 'https://data.commoncrawl.org', sort=False).iter_range_jobs()) - assert [j.url for j in jobs] == [ - 'https://data.commoncrawl.org/crawl-data/x.warc.gz', 'https://example.org/y.warc.gz', - ] - assert jobs[0].filename == 'crawl-data/x.warc.gz' - - -def test_writer_extra_columns_round_trip(tmp_path): - path = str(tmp_path / 'ranges.csv') - w = RangeJobCsvWriter(path) - w.write(RangeJob( - url='https://data.commoncrawl.org/crawl-data/x.warc.gz', - offset=10, length=20, filename='crawl-data/x.warc.gz', - extra={'content_languages': 'eng', 'url': 'https://example.com/page'}, - )) - w.close() - - with open(path, newline='') as f: - reader = csv.DictReader(f) - assert reader.fieldnames == [ - 'warc_filename', 'warc_record_offset', 'warc_record_length', - 'content_languages', 'url', - ] - rows = list(reader) - assert rows[0]['content_languages'] == 'eng' - assert rows[0]['url'] == 'https://example.com/page' - - # extras round-trip back onto RangeJob.extra - jobs = list(CsvSource(path, None).iter_range_jobs()) - assert jobs[0].extra == {'content_languages': 'eng', 'url': 'https://example.com/page'} - assert jobs[0].url == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' - - -def test_writer_empty_writes_base_header(tmp_path): - path = str(tmp_path / 'empty.csv') - w = RangeJobCsvWriter(path) - w.close() - with open(path, newline='') as f: - reader = csv.DictReader(f) - rows = list(reader) - assert reader.fieldnames == ['warc_filename', 'warc_record_offset', 'warc_record_length'] - assert rows == [] - - -def test_csv_source_warc_url_column_is_not_supported(tmp_path): - path = str(tmp_path / 'old.csv') - with open(path, 'w') as f: - f.write('warc_url,warc_record_offset,warc_record_length\n') - f.write('s3://commoncrawl/crawl-data/x.warc.gz,5,7\n') - with pytest.raises(ValueError, match='warc_filename'): - list(CsvSource(path, 'https://x').iter_range_jobs()) - - -def test_csv_source_missing_columns(tmp_path): - path = str(tmp_path / 'bad.csv') - with open(path, 'w') as f: - f.write('foo,bar\n1,2\n') - with pytest.raises(ValueError): - list(CsvSource(path, 'https://x').iter_range_jobs()) - - -def test_csv_source_tsv(tmp_path): - path = str(tmp_path / 'ranges.tsv') - with open(path, 'w') as f: - f.write('warc_filename\twarc_record_offset\twarc_record_length\n') - f.write('crawl-data/x.warc.gz\t10\t20\n') - jobs = list(CsvSource(path, 'https://data.commoncrawl.org').iter_range_jobs()) - assert jobs[0].url == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' - assert jobs[0].offset == 10 and jobs[0].length == 20 - - -def test_csv_source_sorts_by_filename_then_offset(tmp_path): - path = str(tmp_path / 'ranges.csv') - with open(path, 'w') as f: - f.write('warc_filename,warc_record_offset,warc_record_length\n') - # deliberately out of order: file b before a, and offsets descending - f.write('b.warc.gz,50,1\n') - f.write('a.warc.gz,200,1\n') - f.write('a.warc.gz,10,1\n') - jobs = list(CsvSource(path, 'https://data.commoncrawl.org').iter_range_jobs()) - assert [(j.filename, j.offset) for j in jobs] == [ - ('a.warc.gz', 10), ('a.warc.gz', 200), ('b.warc.gz', 50), - ] - - -def test_csv_source_no_sort_preserves_order(tmp_path): - path = str(tmp_path / 'ranges.csv') - with open(path, 'w') as f: - f.write('warc_filename,warc_record_offset,warc_record_length\n') - f.write('b.warc.gz,50,1\n') - f.write('a.warc.gz,200,1\n') - f.write('a.warc.gz,10,1\n') - jobs = list(CsvSource(path, 'https://data.commoncrawl.org', sort=False).iter_range_jobs()) - assert [(j.filename, j.offset) for j in jobs] == [ - ('b.warc.gz', 50), ('a.warc.gz', 200), ('a.warc.gz', 10), - ] diff --git a/tests/repackage/test_records_source.py b/tests/repackage/test_records_source.py new file mode 100644 index 0000000..52d90f2 --- /dev/null +++ b/tests/repackage/test_records_source.py @@ -0,0 +1,26 @@ +from cdx_toolkit.repackage.records_source import join_warc_url + + +def test_join_warc_url_relative(): + assert ( + join_warc_url('https://data.commoncrawl.org', 'crawl-data/x.warc.gz') + == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' + ) + + +def test_join_warc_url_no_double_slash(): + assert ( + join_warc_url('https://data.commoncrawl.org/', 'crawl-data/x.warc.gz') + == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' + ) + assert join_warc_url('s3://bucket', '/crawl-data/x.warc.gz') == 's3://bucket/crawl-data/x.warc.gz' + + +def test_join_warc_url_empty_prefix(): + assert join_warc_url('', 'crawl-data/x.warc.gz') == 'crawl-data/x.warc.gz' + assert join_warc_url(None, 'crawl-data/x.warc.gz') == 'crawl-data/x.warc.gz' + + +def test_join_warc_url_absolute_filename_passthrough(): + assert join_warc_url('https://data.commoncrawl.org', 's3://cc/x.warc.gz') == 's3://cc/x.warc.gz' + assert join_warc_url('', 'https://host/x.warc.gz') == 'https://host/x.warc.gz' diff --git a/tests/repackage/test_warc_filter.py b/tests/repackage/test_warc_filter.py index 39a8e21..4f4509b 100644 --- a/tests/repackage/test_warc_filter.py +++ b/tests/repackage/test_warc_filter.py @@ -2,7 +2,7 @@ from tests.conftest import TEST_DATA_PATH from cdx_toolkit.repackage.warc_filter import WARCFilter -from cdx_toolkit.repackage.sources.records import RecordsSource +from cdx_toolkit.repackage.records_source import RecordsSource fixture_path = TEST_DATA_PATH / 'repackage' diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index f56f1d9..64770cb 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -15,7 +15,7 @@ from cdx_toolkit.repackage import multiprocess from cdx_toolkit.repackage.data_classes import RangeJob, RepackageResult from cdx_toolkit.repackage.local_writer import LocalFileWriter -from cdx_toolkit.repackage.sources.records import RecordsSource +from cdx_toolkit.repackage.records_source import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter from tests.fake_warc_server import FakeWarcServer diff --git a/tests/repackage/test_athena_source.py b/tests/unit/test_athena.py similarity index 66% rename from tests/repackage/test_athena_source.py rename to tests/unit/test_athena.py index 84f5395..3aaf8c4 100644 --- a/tests/repackage/test_athena_source.py +++ b/tests/unit/test_athena.py @@ -1,4 +1,4 @@ -"""AthenaSource and the Athena index backend with a fake boto3 Athena client (no AWS).""" +"""Running Athena queries and the Athena index backend with a fake boto3 Athena client (no AWS).""" import json import logging from unittest.mock import patch @@ -8,7 +8,7 @@ from cdx_toolkit.cli import main from cdx_toolkit.indexes import IndexQuery from cdx_toolkit.indexes.sql import AthenaBackend -from cdx_toolkit.repackage.sources.athena import AthenaSource, report_query_cost, run_athena_query +from cdx_toolkit.indexes.athena import iter_athena_rows, make_client, report_query_cost, run_athena_query ATHENA = 'athena://athena.eu-west-1.amazonaws.com/mydb?S3OutputLocation=s3://bucket/athena/' COLUMNS = ['url', 'fetch_time', 'fetch_status', 'warc_filename', 'warc_record_offset', 'warc_record_length'] @@ -57,30 +57,19 @@ def get_paginator(self, name): @pytest.fixture def client(): fake = FakeAthenaClient() - with patch.object(AthenaSource, '_make_client', return_value=fake), \ - patch('cdx_toolkit.repackage.sources.athena.time.sleep'): + with patch('cdx_toolkit.indexes.athena.make_client', return_value=fake), \ + patch('cdx_toolkit.indexes.athena.time.sleep'): yield fake -def test_source_rows_jobs_and_cost(client, caplog): - source = AthenaSource(query='SELECT 1', database='ccindex', s3_output_location='s3://b/p/', - warc_download_prefix='s3://commoncrawl') - +def test_rows_and_cost(client, caplog): with caplog.at_level(logging.INFO): - rows = list(source.iter_rows()) + rows = list(iter_athena_rows('SELECT 1', 'ccindex', 's3://b/p/')) assert rows == [dict(zip(COLUMNS, row)) for row in ROWS] assert client.started[0]['QueryExecutionContext'] == {'Database': 'ccindex'} assert client.started[0]['ResultConfiguration'] == {'OutputLocation': 's3://b/p/'} assert 'Athena scanned 2.00 GB, estimated cost ~$0.0100' in caplog.text - jobs = list(source.iter_range_jobs()) - assert [(j.url, j.offset, j.length) for j in jobs] == [ - ('s3://commoncrawl/crawl-data/a.warc.gz', 10, 100), ('s3://commoncrawl/crawl-data/b.warc.gz', 20, 200), - ] - assert jobs[0].extra == { - 'url': 'https://example.com/', 'fetch_time': '2024-07-13 10:22:33.000', 'fetch_status': '200', - } - def test_index_athena_end_to_end(client, capsys): main(['index', ATHENA, '--hostname', 'example.com', '--yes']) @@ -100,8 +89,8 @@ def test_athena_count(): fake.get_paginator = lambda name: type('P', (), {'paginate': lambda self, **kw: iter([ {'ResultSet': {'Rows': [{'Data': [{'VarCharValue': 'n'}]}, {'Data': [{'VarCharValue': '42'}]}]}} ])})() - with patch.object(AthenaSource, '_make_client', return_value=fake), \ - patch('cdx_toolkit.repackage.sources.athena.time.sleep'): + with patch('cdx_toolkit.indexes.athena.make_client', return_value=fake), \ + patch('cdx_toolkit.indexes.athena.time.sleep'): assert AthenaBackend(ATHENA, assume_yes=True).count(IndexQuery(domains=['example.com'])) == 42 assert fake.started[0]['QueryString'].startswith('SELECT count(*) AS n FROM (') @@ -116,7 +105,7 @@ def test_failed_query(caplog): def test_query_timeout_is_cancelled(): fake = FakeAthenaClient(states=['RUNNING']) - with patch('cdx_toolkit.repackage.sources.athena.time.sleep'), pytest.raises(TimeoutError): + with patch('cdx_toolkit.indexes.athena.time.sleep'), pytest.raises(TimeoutError): run_athena_query(fake, 'SELECT 1', 'ccindex', 's3://b/p/', max_wait_time=0) assert fake.stopped == ['qid'] @@ -132,11 +121,60 @@ def test_make_client_uses_the_region(monkeypatch): monkeypatch.delenv('AWS_MAX_ATTEMPTS', raising=False) monkeypatch.delenv('AWS_RETRY_MODE', raising=False) monkeypatch.setattr('cdx_toolkit.aws.user_aws_config', lambda: {}) - source = AthenaSource(query='SELECT 1', database='d', s3_output_location='s3://b/', warc_download_prefix=None, - region_name='eu-west-1') with patch('boto3.client') as boto3_client: - source._make_client() + make_client('eu-west-1') args, kwargs = boto3_client.call_args assert args == ('athena',) and kwargs['config'].region_name == 'eu-west-1' # Common Crawl's recommended retry settings (cdx_toolkit.aws) assert kwargs['config'].retries == {'mode': 'adaptive', 'total_max_attempts': 100000} + + +class _FakeAthenaClient: + """Minimal stand-in for a boto3 Athena client (no AWS, no network).""" + + def __init__(self, raise_on_wait=None): + self._raise_on_wait = raise_on_wait + self.started = [] + self.stopped = [] + + def start_query_execution(self, **kwargs): + self.started.append(kwargs) + return {'QueryExecutionId': 'qid-123'} + + def get_query_execution(self, QueryExecutionId): + if self._raise_on_wait is not None: + raise self._raise_on_wait + return {'QueryExecution': {'Status': {'State': 'SUCCEEDED'}, + 'Statistics': {'DataScannedInBytes': 1000}}} + + def stop_query_execution(self, QueryExecutionId): + self.stopped.append(QueryExecutionId) + + +def test_run_athena_query_logs_sql(caplog): + import logging + client = _FakeAthenaClient() + with caplog.at_level(logging.INFO): + run_athena_query(client, 'SELECT my_special_column FROM ccindex', 'ccindex', 's3://b/out/') + assert 'SELECT my_special_column FROM ccindex' in caplog.text + + +def test_run_athena_query_success_no_cancel(): + client = _FakeAthenaClient() + qid = run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') + assert qid == 'qid-123' + assert client.stopped == [] + + +def test_run_athena_query_cancels_on_interrupt(): + client = _FakeAthenaClient(raise_on_wait=KeyboardInterrupt()) + with pytest.raises(KeyboardInterrupt): + run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') + assert client.stopped == ['qid-123'] + + +def test_run_athena_query_cancels_on_timeout(): + client = _FakeAthenaClient(raise_on_wait=TimeoutError('too slow')) + with pytest.raises(TimeoutError): + run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') + assert client.stopped == ['qid-123'] diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py index 3623651..c133dec 100644 --- a/tests/unit/test_aws.py +++ b/tests/unit/test_aws.py @@ -129,7 +129,7 @@ def test_s3fs_uses_the_defaults(no_user_config, monkeypatch): def test_warc_fetcher_s3_clients(no_user_config): pytest.importorskip('aioboto3') - from cdx_toolkit.repackage.sources.records import RecordsSource + from cdx_toolkit.repackage.records_source import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter wf = WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=4) diff --git a/tests/repackage/test_athena_prompt.py b/tests/unit/test_confirm_cost.py similarity index 65% rename from tests/repackage/test_athena_prompt.py rename to tests/unit/test_confirm_cost.py index 826640c..4f4d19e 100644 --- a/tests/repackage/test_athena_prompt.py +++ b/tests/unit/test_confirm_cost.py @@ -3,29 +3,21 @@ import pytest from cdx_toolkit.indexes.sql import confirm_cost -from cdx_toolkit.repackage.sources.base import CostEstimate def est(n_crawls): - return CostEstimate(n_crawls=n_crawls, engine='athena') - - -def test_none_estimate_never_prompts(): - # cdx/csv sources return None -> no cost prompt - with patch('builtins.input') as inp: - confirm_cost(None, confirmed=False) - inp.assert_not_called() + return n_crawls, 'athena' def test_small_crawl_set_no_prompt(): with patch('builtins.input') as inp: - confirm_cost(est(5), confirmed=False) + confirm_cost(*est(5), confirmed=False) inp.assert_not_called() def test_confirmed_flag_bypasses_prompt(): with patch('builtins.input') as inp: - confirm_cost(est(None), confirmed=True) + confirm_cost(*est(None), confirmed=True) inp.assert_not_called() @@ -33,31 +25,31 @@ def test_large_crawl_set_non_tty_aborts(): with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin: stdin.isatty.return_value = False with pytest.raises(SystemExit): - confirm_cost(est(11), confirmed=False) + confirm_cost(*est(11), confirmed=False) def test_unknown_crawls_non_tty_aborts(): with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin: stdin.isatty.return_value = False with pytest.raises(SystemExit): - confirm_cost(est(None), confirmed=False) + confirm_cost(*est(None), confirmed=False) def test_tty_yes_proceeds(): with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch('builtins.input', return_value='y'): stdin.isatty.return_value = True - confirm_cost(est(None), confirmed=False) # should not raise + confirm_cost(*est(None), confirmed=False) # should not raise def test_tty_empty_answer_aborts(): with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch('builtins.input', return_value=''): stdin.isatty.return_value = True with pytest.raises(SystemExit): - confirm_cost(est(None), confirmed=False) + confirm_cost(*est(None), confirmed=False) def test_tty_no_answer_aborts(): with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch('builtins.input', return_value='n'): stdin.isatty.return_value = True with pytest.raises(SystemExit): - confirm_cost(est(11), confirmed=False) + confirm_cost(*est(11), confirmed=False) diff --git a/tests/repackage/test_duckdb_source.py b/tests/unit/test_duckdb.py similarity index 62% rename from tests/repackage/test_duckdb_source.py rename to tests/unit/test_duckdb.py index 9c934bb..fa31d70 100644 --- a/tests/repackage/test_duckdb_source.py +++ b/tests/unit/test_duckdb.py @@ -1,49 +1,42 @@ -"""DuckDbSource with a local parquet file, and its connection setup with a fake connection (no network).""" +"""DuckDB queries of a local parquet file, and the connection setup with a fake connection (no network).""" from unittest.mock import MagicMock, patch import pytest -from cdx_toolkit.repackage.sources import duckdb as duckdb_source -from cdx_toolkit.repackage.sources.duckdb import DuckDbSource, _build_from_clause +from cdx_toolkit.indexes import duckdb as duckdb_source +from cdx_toolkit.indexes.duckdb import build_from_clause, iter_duckdb_rows from tests.conftest import requires_duckdb def test_from_clause(): - assert _build_from_clause('s3://b/index/', ['CC-MAIN-2024-33']) == ( + assert build_from_clause('s3://b/index/', ['CC-MAIN-2024-33']) == ( "read_parquet(['s3://b/index/crawl=CC-MAIN-2024-33/subset=warc/*.parquet'], hive_partitioning=true)" ) - assert _build_from_clause('s3://b/index', None) == ( + assert build_from_clause('s3://b/index', None) == ( "read_parquet(['s3://b/index/crawl=*/subset=warc/*.parquet'], hive_partitioning=true)" ) @requires_duckdb -def test_range_jobs_from_parquet(tmp_path): +def test_rows_from_parquet(tmp_path): import duckdb path = tmp_path / 'index.parquet' duckdb.connect().execute(f''' - COPY (SELECT * FROM (VALUES ('crawl-data/a.warc.gz', 10, 100, 'https://example.com/'), - ('crawl-data/b.warc.gz', 20, 200, 'https://example.com/x')) - AS t(warc_filename, warc_record_offset, warc_record_length, url)) + COPY (SELECT * FROM (VALUES ('crawl-data/a.warc.gz', 10), ('crawl-data/b.warc.gz', 20)) + AS t(warc_filename, warc_record_offset)) TO '{path}' (FORMAT parquet) ''') - - source = DuckDbSource(query=f"SELECT * FROM read_parquet('{path}')", warc_download_prefix='s3://commoncrawl') - jobs = list(source.iter_range_jobs()) - assert [(j.url, j.offset, j.length, j.extra) for j in jobs] == [ - ('s3://commoncrawl/crawl-data/a.warc.gz', 10, 100, {'url': 'https://example.com/'}), - ('s3://commoncrawl/crawl-data/b.warc.gz', 20, 200, {'url': 'https://example.com/x'}), + assert list(iter_duckdb_rows(f"SELECT * FROM read_parquet('{path}')")) == [ + {'warc_filename': 'crawl-data/a.warc.gz', 'warc_record_offset': 10}, + {'warc_filename': 'crawl-data/b.warc.gz', 'warc_record_offset': 20}, ] - with pytest.raises(ValueError, match='missing required columns'): - list(DuckDbSource(query=f"SELECT url FROM read_parquet('{path}')").iter_range_jobs()) - def test_without_duckdb(): with patch.object(duckdb_source, '_HAS_DUCKDB', False), pytest.raises(RuntimeError, match=r'cdx_toolkit\[duckdb\]'): - list(DuckDbSource(query='SELECT 1').iter_rows()) + list(iter_duckdb_rows('SELECT 1')) def test_connection_setup_and_hf_filesystem(monkeypatch): @@ -61,7 +54,7 @@ def test_connection_setup_and_hf_filesystem(monkeypatch): with patch.object(duckdb_source, '_HAS_DUCKDB', True), \ patch.object(duckdb_source, 'duckdb', fake_duckdb, create=True), \ patch.dict('sys.modules', {'huggingface_hub': fake_hf}): - rows = list(DuckDbSource(query=query).iter_rows()) + rows = list(iter_duckdb_rows(query)) assert rows == [{'n': 1}] statements = [c.args[0] for c in con.execute.call_args_list] diff --git a/tests/unit/test_sql_query.py b/tests/unit/test_sql_query.py new file mode 100644 index 0000000..00fdbea --- /dev/null +++ b/tests/unit/test_sql_query.py @@ -0,0 +1,81 @@ +"""Building the guided SQL queries for the columnar index (no engine).""" +import pytest + +from cdx_toolkit.indexes.sql_query import build_sql, escape_sql_literal + + +def test_build_query_hostnames(): + q = build_sql('ccindex', ['example.com', 'test.org']) + assert "subset = 'warc'" in q + assert "url_host_tld = 'com'" in q + assert "url_host_tld = 'org'" in q + assert "url_host_name = 'example.com'" in q + assert "url_host_name = 'test.org'" in q + assert 'warc_filename, warc_record_offset, warc_record_length' in q + assert 'crawl IN' not in q + assert 'LIMIT' not in q + + +def test_build_query_domains_only(): + q = build_sql('ccindex', url_host_registered_domains=['example.com']) + assert "url_host_registered_domain IN ('example.com')" in q + assert "url_host_tld = 'com'" in q + assert 'url_host_name' not in q + + +def test_build_query_hostnames_and_domains(): + q = build_sql('ccindex', ['www.example.com'], url_host_registered_domains=['example.org']) + assert "url_host_name = 'www.example.com'" in q + assert "url_host_registered_domain IN ('example.org')" in q + assert "url_host_tld = 'com'" in q + assert "url_host_tld = 'org'" in q + + +def test_build_query_requires_host_or_domain(): + with pytest.raises(ValueError): + build_sql('ccindex') + + +def test_build_query_with_crawls(): + q = build_sql('ccindex', ['example.com'], crawls=['CC-MAIN-2025-33', 'CC-MAIN-2025-30']) + assert "crawl IN ('CC-MAIN-2025-33', 'CC-MAIN-2025-30')" in q + + +def test_build_query_limit(): + assert 'LIMIT 10' in build_sql('ccindex', ['example.com'], limit=10) + assert 'LIMIT' not in build_sql('ccindex', ['example.com'], limit=0) + + +def test_build_query_orders_by_default(): + q = build_sql('ccindex', ['example.com']) + assert 'ORDER BY warc_filename, warc_record_offset' in q + + +def test_build_query_order_by_false(): + assert 'ORDER BY' not in build_sql('ccindex', ['example.com'], order_by=False) + + +def test_build_query_order_by_precedes_limit(): + q = build_sql('ccindex', ['example.com'], limit=10) + assert q.index('ORDER BY') < q.index('LIMIT') + + +def test_build_query_requires_hostnames(): + with pytest.raises(ValueError): + build_sql('ccindex', []) + + +def test_escape_sql_literal_valid(): + assert escape_sql_literal('example.com') == "'example.com'" + assert escape_sql_literal('CC-MAIN-2025-33') == "'CC-MAIN-2025-33'" + + +@pytest.mark.parametrize('bad', ["a';DROP TABLE x;--", "a' OR '1'='1", 'a b', "a'", "', '"]) +def test_escape_sql_literal_rejects_injection(bad): + with pytest.raises(ValueError): + escape_sql_literal(bad) + + +def test_build_query_rejects_injection_hostname(): + with pytest.raises(ValueError): + build_sql('ccindex', ["example.com'; DROP TABLE ccindex; --"]) diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py index 291a07f..28622c5 100644 --- a/tests/unit/test_usage_errors.py +++ b/tests/unit/test_usage_errors.py @@ -15,7 +15,7 @@ from cdx_toolkit.indexes.sql import DuckDbBackend, sql_timestamp from cdx_toolkit.record_io import RecordWriter, import_pyarrow, read_records, write_records from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.sources.sql_base import quote_sql_string +from cdx_toolkit.indexes.sql_query import quote_sql_string from tests.conftest import TEST_DATA_PATH From 3a279994dc49080a34cf1e692c3741c978d994ac Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 16:33:59 +0200 Subject: [PATCH 20/91] test: replace repetitive writer tests with behaviour tests, cover the S3 and metadata paths offline - the local and S3 writer tests check behaviour (buffering, exact bytes, part sizes, close, abort) with a fake S3 client instead of repeating constructor assignments - offline tests for S3 output with metadata records, a failing metadata record, S3 read lengths, crawl name resolution and URL filters with one output per CDX file - LocalFileWriter.close closes the file in a finally --- cdx_toolkit/repackage/local_writer.py | 6 +- tests/repackage/test_local_writer.py | 488 ++----------------- tests/repackage/test_s3_writer.py | 281 ++--------- tests/repackage/test_warc_filter_pipeline.py | 63 ++- tests/unit/test_cdx_files.py | 15 + tests/unit/test_sql_query.py | 11 + 6 files changed, 178 insertions(+), 686 deletions(-) diff --git a/cdx_toolkit/repackage/local_writer.py b/cdx_toolkit/repackage/local_writer.py index 69e365b..f13aa25 100644 --- a/cdx_toolkit/repackage/local_writer.py +++ b/cdx_toolkit/repackage/local_writer.py @@ -38,12 +38,10 @@ async def close(self): try: if self.buffer: await self._flush() + finally: if self.file_handle: await self.file_handle.close() - except Exception: - if self.file_handle: - await self.file_handle.close() - raise + self.file_handle = None async def abort(self): """Close and remove the (incomplete) file.""" diff --git a/tests/repackage/test_local_writer.py b/tests/repackage/test_local_writer.py index c1f3190..1e0650d 100644 --- a/tests/repackage/test_local_writer.py +++ b/tests/repackage/test_local_writer.py @@ -1,474 +1,72 @@ import asyncio +import os + import pytest -from unittest.mock import patch from cdx_toolkit.repackage.local_writer import LocalFileWriter -def test_init_default_values(): - """Test initialization with default values.""" - writer = LocalFileWriter('/tmp/test.txt') - assert writer.file_path == '/tmp/test.txt' - assert writer.buffer_size == 8192 - assert writer.mode == 'wb' - assert writer.file_handle is None - assert isinstance(writer.buffer, bytearray) - assert len(writer.buffer) == 0 - - -def test_init_custom_values(): - """Test initialization with custom values.""" - writer = LocalFileWriter('/tmp/test.txt', buffer_size=4096, mode='ab') - assert writer.file_path == '/tmp/test.txt' - assert writer.buffer_size == 4096 - assert writer.mode == 'ab' - assert writer.file_handle is None - assert isinstance(writer.buffer, bytearray) - assert len(writer.buffer) == 0 - - -def test_start_opens_file(tmp_path): - """Test that start() opens the file correctly.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) - await writer.start() - - assert writer.file_handle is not None - await writer.close() - - asyncio.run(run_test()) - - -def test_start_with_different_modes(tmp_path): - """Test start() with different file modes.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - - # Test binary write mode - writer = LocalFileWriter(str(temp_file), mode='wb') - await writer.start() - assert writer.file_handle is not None - await writer.close() - - # Test binary append mode - writer = LocalFileWriter(str(temp_file), mode='ab') - await writer.start() - assert writer.file_handle is not None - await writer.close() - - asyncio.run(run_test()) - - -def test_start_creates_directory_if_needed(tmp_path): - """Test that start() works when parent directory exists.""" - - async def run_test(): - subdir = tmp_path / 'subdir' - subdir.mkdir() - temp_file = subdir / 'test.txt' - - writer = LocalFileWriter(str(temp_file)) - await writer.start() - assert writer.file_handle is not None - await writer.close() - - asyncio.run(run_test()) - - -def test_write_small_data_buffers(tmp_path): - """Test writing data that doesn't exceed buffer size.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file), buffer_size=100) - await writer.start() - - test_data = b'Hello, World!' - await writer.write(test_data) - - # Data should be in buffer, not yet written to file - assert len(writer.buffer) == len(test_data) - assert writer.buffer == test_data - - await writer.close() - - # After close, data should be written to file - assert temp_file.read_bytes() == test_data - - asyncio.run(run_test()) - - -def test_write_large_data_triggers_flush(tmp_path): - """Test writing data that exceeds buffer size triggers flush.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - buffer_size = 50 - writer = LocalFileWriter(str(temp_file), buffer_size=buffer_size) - await writer.start() - - # Write data larger than buffer size - test_data = b'x' * (buffer_size + 10) - await writer.write(test_data) - - # Buffer should be empty after automatic flush - assert len(writer.buffer) == 0 - - await writer.close() - - # Data should be written to file - assert temp_file.read_bytes() == test_data - - asyncio.run(run_test()) - - -def test_write_multiple_small_chunks(tmp_path): - """Test writing multiple small chunks that eventually trigger flush.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - buffer_size = 50 - writer = LocalFileWriter(str(temp_file), buffer_size=buffer_size) - await writer.start() - - chunk1 = b'a' * 30 - chunk2 = b'b' * 25 # Total: 55 bytes, exceeds buffer - - await writer.write(chunk1) - assert len(writer.buffer) == 30 - - await writer.write(chunk2) - # Should have triggered flush, buffer should be empty - assert len(writer.buffer) == 0 - - await writer.close() - - assert temp_file.read_bytes() == chunk1 + chunk2 - - asyncio.run(run_test()) - - -def test_write_empty_data(tmp_path): - """Test writing empty data.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) - await writer.start() - - await writer.write(b'') - assert len(writer.buffer) == 0 - - await writer.close() - - assert temp_file.read_bytes() == b'' - - asyncio.run(run_test()) - - -def test_write_without_start_graceful_handling(tmp_path): - """Test that writing without calling start() is handled gracefully.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file), buffer_size=10) # Small buffer to force flush - - # This should work fine as long as we don't exceed buffer size - await writer.write(b'small') - assert len(writer.buffer) == 5 - - # When buffer exceeds size, flush is called but does nothing since file_handle is None - # The data stays in buffer instead of being written - await writer.write(b'data that exceeds buffer size') - - # Buffer should contain all the data since flush did nothing - expected_data = b'small' + b'data that exceeds buffer size' - assert writer.buffer == expected_data - - asyncio.run(run_test()) - - -def test_flush_empty_buffer(tmp_path): - """Test flushing when buffer is empty.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) - await writer.start() - - # Flush empty buffer should not raise error - await writer._flush() - assert len(writer.buffer) == 0 - - await writer.close() - - asyncio.run(run_test()) - - -def test_flush_without_file_handle(tmp_path): - """Test flushing without file handle.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) - writer.buffer.extend(b'test data') - - # Should not raise error, just do nothing - await writer._flush() - assert len(writer.buffer) == len(b'test data') # Buffer unchanged +def run(coro): + return asyncio.run(coro) - asyncio.run(run_test()) +def test_writes_buffered_and_byte_exact(tmp_path): + path = tmp_path / 'out.warc.gz' + chunks = [bytes(range(256)), b'small', b'x' * 20000, b'', b'\x00\xff' * 10] -def test_close_flushes_remaining_data(tmp_path): - """Test that close() flushes any remaining buffered data.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file), buffer_size=100) + async def write(): + writer = LocalFileWriter(str(path), buffer_size=1024) await writer.start() - - test_data = b'This data should be flushed on close' - await writer.write(test_data) - - # Data should still be in buffer - assert len(writer.buffer) == len(test_data) - + await writer.write(chunks[1]) + assert path.read_bytes() == b'' # buffered + await writer.write(chunks[2]) + assert len(path.read_bytes()) == len(chunks[1] + chunks[2]) # flushed at the buffer size + for chunk in chunks[3:]: + await writer.write(chunk) await writer.close() - # Data should now be written to file - assert temp_file.read_bytes() == test_data - - asyncio.run(run_test()) + run(write()) + assert path.read_bytes() == b''.join(chunks[1:]) def test_close_without_start(tmp_path): - """Test closing without calling start().""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) - - # Should not raise error - await writer.close() - - asyncio.run(run_test()) - - -def test_close_twice(tmp_path): - """Test calling close() multiple times.""" + run(LocalFileWriter(str(tmp_path / 'out')).close()) + assert not (tmp_path / 'out').exists() - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) - await writer.start() - - await writer.close() - - # Second close should not raise error - await writer.close() - - asyncio.run(run_test()) +def test_abort_removes_the_file(tmp_path): + path = tmp_path / 'out.warc.gz' -def test_close_handles_flush_exception(tmp_path): - """Test that close() handles exceptions during flush properly.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) + async def write_and_abort(): + writer = LocalFileWriter(str(path), buffer_size=4) await writer.start() + await writer.write(b'incomplete') + await writer.abort() + await writer.abort() # idempotent - # Add some data to buffer - await writer.write(b'test data') - - # Mock flush to raise an exception - with patch.object(writer, '_flush', side_effect=Exception('Flush error')): - with pytest.raises(Exception, match='Flush error'): - await writer.close() - - asyncio.run(run_test()) - - -def test_close_handles_file_close_exception(tmp_path): - """Test that close() handles exceptions during file close.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) - await writer.start() - - # Mock file handle close to raise an exception - with patch.object(writer.file_handle, 'close', side_effect=Exception('Close error')): - with pytest.raises(Exception, match='Close error'): - await writer.close() - - asyncio.run(run_test()) - - -def test_large_file_write(tmp_path): - """Test writing a large amount of data.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file), buffer_size=1024) - await writer.start() - - # Write 1MB of data in chunks - chunk_size = 1024 # Make chunk size same as buffer for exact division - total_size = 1024 * 1024 # 1MB - chunk_data = b'x' * chunk_size - - for _ in range(total_size // chunk_size): - await writer.write(chunk_data) - - await writer.close() - - # Verify file size - assert temp_file.stat().st_size == total_size - - asyncio.run(run_test()) - - -def test_binary_data_integrity(tmp_path): - """Test that binary data is written correctly.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) - await writer.start() - - # Create binary data with all byte values - binary_data = bytes(range(256)) - await writer.write(binary_data) - - await writer.close() - - assert temp_file.read_bytes() == binary_data + run(write_and_abort()) + assert not path.exists() + run(LocalFileWriter(str(path)).abort()) # without start - asyncio.run(run_test()) +def test_start_fails_without_the_directory(tmp_path): + with pytest.raises(FileNotFoundError): + run(LocalFileWriter(os.path.join(str(tmp_path), 'missing', 'out')).start()) -def test_concurrent_writes(tmp_path): - """Test concurrent write operations.""" - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file), buffer_size=100) +def test_close_closes_the_file_if_the_last_write_fails(tmp_path): + async def close_with_failing_flush(): + writer = LocalFileWriter(str(tmp_path / 'out')) await writer.start() + await writer.write(b'x') + handle = writer.file_handle - # Create multiple write tasks - async def write_chunk(data): - await writer.write(data) - - tasks = [write_chunk(f'chunk{i}'.encode() * 10) for i in range(10)] - - await asyncio.gather(*tasks) - await writer.close() - - # Verify file exists and has data - assert temp_file.exists() - assert temp_file.stat().st_size > 0 - - asyncio.run(run_test()) - - -def test_file_permissions_error(tmp_path): - """Test handling of file permission errors.""" - - async def run_test(): - # Create a file path in a directory we can't write to - readonly_file = tmp_path / 'readonly.txt' - - # Create the file first - readonly_file.write_text('test') - - # Make the file read-only - readonly_file.chmod(0o444) - - writer = LocalFileWriter(str(readonly_file), mode='wb') - - with pytest.raises(PermissionError): - await writer.start() - - asyncio.run(run_test()) - - -def test_nonexistent_directory(): - """Test writing to a file in a nonexistent directory.""" - - async def run_test(): - nonexistent_path = '/nonexistent/directory/file.txt' - writer = LocalFileWriter(nonexistent_path) - - with pytest.raises(FileNotFoundError): - await writer.start() - - asyncio.run(run_test()) - - -def test_context_manager_like_usage(tmp_path): - """Test typical usage pattern similar to context manager.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - writer = LocalFileWriter(str(temp_file)) + async def fail(): + raise OSError('disk full') - try: - await writer.start() - await writer.write(b'Hello, World!') - await writer.write(b' How are you?') - finally: + writer._flush = fail + with pytest.raises(OSError): await writer.close() + assert writer.file_handle is None and handle.closed - assert temp_file.read_bytes() == b'Hello, World! How are you?' - - asyncio.run(run_test()) - - -def test_buffer_size_edge_cases(tmp_path): - """Test edge cases with different buffer sizes.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - # Test with buffer size of 1 - writer = LocalFileWriter(str(temp_file), buffer_size=1) - await writer.start() - - await writer.write(b'a') # Should trigger flush immediately - assert len(writer.buffer) == 0 - - await writer.write(b'bc') # Should trigger flush after 'b', leaving 'c' - assert len(writer.buffer) == 0 - - await writer.close() - - assert temp_file.read_bytes() == b'abc' - - asyncio.run(run_test()) - - -def test_append_mode(tmp_path): - """Test append mode functionality.""" - - async def run_test(): - temp_file = tmp_path / 'test.txt' - - # First, write some initial data - temp_file.write_bytes(b'Initial data\n') - - # Now append using LocalFileWriter - writer = LocalFileWriter(str(temp_file), mode='ab') - await writer.start() - - await writer.write(b'Appended data\n') - await writer.close() - - # Verify both pieces of data are present - content = temp_file.read_bytes() - assert content == b'Initial data\nAppended data\n' - - asyncio.run(run_test()) + run(close_with_failing_flush()) diff --git a/tests/repackage/test_s3_writer.py b/tests/repackage/test_s3_writer.py index 209109e..97b369d 100644 --- a/tests/repackage/test_s3_writer.py +++ b/tests/repackage/test_s3_writer.py @@ -1,242 +1,9 @@ -import pytest - import asyncio from unittest.mock import AsyncMock -from unittest.mock import patch - -from cdx_toolkit.repackage.s3_writer import mpu_abort - - -from cdx_toolkit.repackage.s3_writer import S3ShardWriter - - -def test_shard_writer_start(): - """Test ShardWriter start method.""" - - async def run_test(): - with patch('cdx_toolkit.repackage.s3_writer.mpu_create') as mock_mpu_create: - mock_mpu_create.return_value = 'test-upload-id' - - mock_s3 = AsyncMock() - - writer = S3ShardWriter( - s3_client=mock_s3, - shard_key='test.warc.gz', - dest_bucket='test-bucket', - content_type='application/gzip', - min_part_size=1024, - max_attempts=3, - base_backoff_seconds=0.1, - ) - - await writer.start() - - assert writer.upload_id == 'test-upload-id' - mock_mpu_create.assert_called_once_with( - mock_s3, - 'test-bucket', - 'test.warc.gz', - max_attempts=3, - base_backoff_seconds=0.1, - ) - - asyncio.run(run_test()) - - -def test_shard_writer_write_small_data(): - """Test ShardWriter write method with small data that stays in buffer.""" - - async def run_test(): - mock_s3 = AsyncMock() - - writer = S3ShardWriter( - s3_client=mock_s3, - shard_key='test.warc.gz', - dest_bucket='test-bucket', - content_type='application/gzip', - min_part_size=1024, # 1 KiB - max_attempts=3, - base_backoff_seconds=0.1, - ) - - small_data = b'small test data' - - await writer.write(small_data) - - # Data should be in buffer, no parts uploaded yet - assert len(writer.buffer) == len(small_data) - assert bytes(writer.buffer) == small_data - assert writer.part_number == 1 - assert len(writer.parts) == 0 - asyncio.run(run_test()) - - -def test_shard_writer_write_large_data(): - """Test ShardWriter write method with large data that triggers part uploads.""" - - async def run_test(): - with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part: - mock_upload_part.return_value = 'test-etag-1' - - mock_s3 = AsyncMock() - - writer = S3ShardWriter( - s3_client=mock_s3, - shard_key='test.warc.gz', - dest_bucket='test-bucket', - content_type='application/gzip', - min_part_size=100, # 100 bytes - max_attempts=3, - base_backoff_seconds=0.1, - ) - writer.upload_id = 'test-upload-id' - - large_data = b'x' * 250 # 250 bytes, should create 2 parts - - await writer.write(large_data) - - # Should have uploaded 2 parts (100 bytes each) with 50 bytes remaining in buffer - assert mock_upload_part.call_count == 2 - assert len(writer.parts) == 2 - assert writer.part_number == 3 # Next part would be #3 - assert len(writer.buffer) == 50 # Remaining bytes - assert bytes(writer.buffer) == b'x' * 50 - - # Verify parts structure - assert writer.parts[0] == {'PartNumber': 1, 'ETag': 'test-etag-1'} - assert writer.parts[1] == {'PartNumber': 2, 'ETag': 'test-etag-1'} - - asyncio.run(run_test()) - - -def test_shard_writer_close_with_buffer(): - """Test ShardWriter close method with data remaining in buffer.""" - - async def run_test(): - with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part, patch( - 'cdx_toolkit.repackage.s3_writer.mpu_complete' - ) as mock_complete: - mock_upload_part.return_value = 'final-etag' - - mock_s3 = AsyncMock() - - writer = S3ShardWriter( - s3_client=mock_s3, - shard_key='test.warc.gz', - dest_bucket='test-bucket', - content_type='application/gzip', - min_part_size=1000, # Large min size to keep data in buffer - max_attempts=3, - base_backoff_seconds=0.1, - ) - writer.upload_id = 'test-upload-id' - - # Add some data to buffer - remaining_data = b'final chunk data' - writer.buffer.extend(remaining_data) - - await writer.close() - - # Should upload the final part and complete MPU - mock_upload_part.assert_called_once_with( - mock_s3, - 'test-bucket', - 'test.warc.gz', - 'test-upload-id', - 1, # part number - remaining_data, - 3, # max attempts - 0.1, # base backoff - ) - - mock_complete.assert_called_once_with( - mock_s3, - 'test-bucket', - 'test.warc.gz', - 'test-upload-id', - [{'PartNumber': 1, 'ETag': 'final-etag'}], - 3, # max attempts - 0.1, # base backoff - ) - - # Buffer should be cleared - assert len(writer.buffer) == 0 - assert len(writer.parts) == 1 - - asyncio.run(run_test()) - - -def test_shard_writer_close_empty(): - """Test ShardWriter close method with no data (empty buffer, no parts).""" - - async def run_test(): - with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part, patch( - 'cdx_toolkit.repackage.s3_writer.mpu_complete' - ) as mock_complete: - mock_s3 = AsyncMock() - - writer = S3ShardWriter( - s3_client=mock_s3, - shard_key='test.warc.gz', - dest_bucket='test-bucket', - content_type='application/gzip', - min_part_size=1000, - max_attempts=3, - base_backoff_seconds=0.1, - ) - writer.upload_id = 'test-upload-id' - - # No data in buffer, no parts uploaded - await writer.close() - - # Should not upload any parts or complete MPU since there's no data - mock_upload_part.assert_not_called() - mock_complete.assert_not_called() - - # State should remain unchanged - assert len(writer.buffer) == 0 - assert len(writer.parts) == 0 - - asyncio.run(run_test()) - - -def test_shard_writer_close_with_exception(): - """Test ShardWriter close method with exception and abort handling.""" - - async def run_test(): - with patch('cdx_toolkit.repackage.s3_writer.mpu_upload_part') as mock_upload_part, patch( - 'cdx_toolkit.repackage.s3_writer.mpu_complete' - ) as mock_complete, patch('cdx_toolkit.repackage.s3_writer.mpu_abort') as mock_abort: - mock_upload_part.return_value = 'error-etag' - mock_complete.side_effect = Exception('Complete failed') - - mock_s3 = AsyncMock() - - writer = S3ShardWriter( - s3_client=mock_s3, - shard_key='test.warc.gz', - dest_bucket='test-bucket', - content_type='application/gzip', - min_part_size=1000, - max_attempts=3, - base_backoff_seconds=0.1, - ) - writer.upload_id = 'test-upload-id' - - # Add some data to buffer to trigger upload and complete - writer.buffer.extend(b'some data') - - # Should raise the exception after attempting abort - with pytest.raises(Exception, match='Complete failed'): - await writer.close() - - # Should have attempted to upload part and complete, then abort on failure - mock_upload_part.assert_called_once() - mock_complete.assert_called_once() - mock_abort.assert_called_once_with(mock_s3, 'test-bucket', 'test.warc.gz', 'test-upload-id') +import pytest - asyncio.run(run_test()) +from cdx_toolkit.repackage.s3_writer import S3ShardWriter, mpu_abort def test_mpu_abort_success(): @@ -277,12 +44,16 @@ async def run_test(): class FakeS3: """upload_part fails `fail_times` times (ClientError), then stores the part bodies.""" - def __init__(self, fail_times): + def __init__(self, fail_times=0, fail_complete=False): self.fail_times = fail_times + self.fail_complete = fail_complete self.bodies = {} self.completed = None self.aborted = False + async def create_multipart_upload(self, Bucket, Key): + return {'UploadId': 'upload-id'} + async def upload_part(self, PartNumber, Body, **kwargs): if self.fail_times: self.fail_times -= 1 @@ -293,6 +64,8 @@ async def upload_part(self, PartNumber, Body, **kwargs): return {'ETag': f'etag-{PartNumber}'} async def complete_multipart_upload(self, MultipartUpload, **kwargs): + if self.fail_complete: + raise RuntimeError('complete failed') self.completed = b''.join(self.bodies[p['PartNumber']] for p in MultipartUpload['Parts']) async def abort_multipart_upload(self, **kwargs): @@ -337,3 +110,39 @@ async def run(): asyncio.run(run()) assert s3.completed == b'ABCDEFGHIJ' assert not s3.aborted + + +def test_shard_writer_uploads_parts_of_the_minimum_size(): + s3 = FakeS3() + writer = make_writer(s3) + + async def run(): + await writer.start() + await writer.write(b'AB') + assert s3.bodies == {} # buffered + await writer.write(b'CDEFGHIJK') + assert s3.bodies == {1: b'ABCD', 2: b'EFGH'} + await writer.close() + + asyncio.run(run()) + assert s3.bodies[3] == b'IJK' + assert s3.completed == b'ABCDEFGHIJK' + + +def test_shard_writer_without_data_completes_nothing(): + s3 = FakeS3() + asyncio.run(make_writer(s3).close()) + assert s3.completed is None and not s3.aborted + + +def test_shard_writer_failed_complete_aborts_the_upload(): + s3 = FakeS3(fail_complete=True) + writer = make_writer(s3) + + async def run(): + await writer.write(b'AB') + with pytest.raises(RuntimeError, match='complete failed'): + await writer.close() + + asyncio.run(run()) + assert s3.aborted diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 64770cb..acbb4d4 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -2,6 +2,7 @@ import asyncio import contextlib import glob +import io import gzip import itertools import os @@ -14,6 +15,7 @@ from cdx_toolkit.repackage import multiprocess from cdx_toolkit.repackage.data_classes import RangeJob, RepackageResult +from cdx_toolkit.repackage.http_range import RangeError from cdx_toolkit.repackage.local_writer import LocalFileWriter from cdx_toolkit.repackage.records_source import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter @@ -196,7 +198,22 @@ def test_uvloop(monkeypatch, caplog, uvloop_installed): class FakeS3Client: - """aioboto3 S3 client stand-in that serves the files of SERVER.""" + """aioboto3 S3 client stand-in that serves the files of SERVER and stores multipart uploads.""" + + def __init__(self): + self.parts = {} + self.objects = {} + + async def create_multipart_upload(self, Bucket, Key): + return {'UploadId': Key} + + async def upload_part(self, Key, PartNumber, Body, **kwargs): + self.parts[(Key, PartNumber)] = Body + return {'ETag': str(PartNumber)} + + async def complete_multipart_upload(self, Bucket, Key, MultipartUpload, **kwargs): + parts = [self.parts[(Key, part['PartNumber'])] for part in MultipartUpload['Parts']] + self.objects[f's3://{Bucket}/{Key}'] = b''.join(parts) async def __aenter__(self): return self @@ -254,6 +271,50 @@ def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): sorted(c['url'] for c in CAPTURES[:3]) +def test_s3_output_with_metadata_records(tmp_path): + pytest.importorskip('aioboto3') + metadata = tmp_path / 'urls.txt' + metadata.write_text('https://example.com/\n') + s3 = FakeS3Client() + with patch('aioboto3.Session') as session, \ + patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + session.return_value.client.return_value = s3 + result = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path='s3://bucket/out', + writer_info={}, write_paths_as_metadata_records=[str(metadata)]).filter() + assert result.written == 4 + data = s3.objects['s3://bucket/out-001.warc.gz'] + records = [(r.rec_type, r.content_stream().read()) for r in ArchiveIterator(io.BytesIO(data))] + assert [rec_type for rec_type, _ in records] == ['warcinfo', 'metadata'] + ['resource'] * 4 + assert records[1][1] == b'https://example.com/\n' + + +def test_failing_metadata_record_removes_the_file(tmp_path): + wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}, write_paths_as_metadata_records=[str(tmp_path / 'missing.txt')]) + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get), \ + pytest.raises(FileNotFoundError): + wf.filter() + assert glob.glob(str(tmp_path / 'out-*')) == [] + + +def test_s3_reads_check_the_length(): + class Readers: + async def s3(self): + class Client: + async def get_object(self, **kwargs): + class Body: + async def read(self): + return b'short' + return {'Body': Body()} + return Client() + + job = RangeJob(url='s3://commoncrawl/crawl-data/a.warc.gz', offset=0, length=10) + with pytest.raises(RangeError, match='requested 10 bytes'): + asyncio.run(job.ranged_get_bytes(1, 0, readers=Readers())) + with pytest.raises(ValueError, match='requires readers'): + asyncio.run(job.ranged_get_bytes(1, 0)) + + def test_no_s3_client_without_s3(tmp_path): with patch('aioboto3.Session') as session, \ patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py index cc4c2ee..bb3c673 100644 --- a/tests/unit/test_cdx_files.py +++ b/tests/unit/test_cdx_files.py @@ -209,3 +209,18 @@ def test_no_globs_over_http(capsys): with pytest.raises(SystemExit): main(command) assert 'globs do not work with http(s) URLs' in capsys.readouterr().err + + +def test_one_output_per_cdx_file_with_url_filters(tmp_path): + out = tmp_path / 'out' + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '--url-fgrep', '/taxonomy/', '--url-fgrepv', '/term/1', + '-o', str(out)]) + with gzip.open(out / CDX_PATH.name, 'rt') as f: + lines = f.read().splitlines() + + def url(line): + return json.loads(line.split(' ', 2)[2])['url'] + + expected = [line for line in LINES if line.startswith('fr,mnhn,') and '/taxonomy/' in url(line)] + assert lines == [line for line in expected if '/term/1' not in url(line)] + assert lines diff --git a/tests/unit/test_sql_query.py b/tests/unit/test_sql_query.py index 00fdbea..711b7d5 100644 --- a/tests/unit/test_sql_query.py +++ b/tests/unit/test_sql_query.py @@ -79,3 +79,14 @@ def test_escape_sql_literal_rejects_injection(bad): def test_build_query_rejects_injection_hostname(): with pytest.raises(ValueError): build_sql('ccindex', ["example.com'; DROP TABLE ccindex; --"]) + + +def test_resolve_crawl_names(): + from unittest.mock import patch + + from cdx_toolkit.indexes.sql_query import resolve_crawl_names + + endpoints = [f'https://index.commoncrawl.org/CC-MAIN-2024-{week}-index' for week in (30, 33, 38)] + with patch('cdx_toolkit.indexes.sql_query.get_cc_endpoints', return_value=endpoints): + assert resolve_crawl_names('2') == ['CC-MAIN-2024-33', 'CC-MAIN-2024-38'] + assert resolve_crawl_names('CC-MAIN-2024-30') == ['CC-MAIN-2024-30'] From de4313aeaae721a565f699ed4ef5a7cc36e3c2de Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 17:09:26 +0200 Subject: [PATCH 21/91] test: skip the S3 client test without aioboto3 (minimal install) --- tests/repackage/test_warc_filter_pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index acbb4d4..a0dedc0 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -316,6 +316,7 @@ async def read(self): def test_no_s3_client_without_s3(tmp_path): + pytest.importorskip('aioboto3') with patch('aioboto3.Session') as session, \ patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), From 89178ff5b7130d9d807d8655ee3296684a20eb38 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 17:42:22 +0200 Subject: [PATCH 22/91] test: requires_aioboto3 decorator instead of importorskip, like requires_duckdb --- tests/conftest.py | 13 +++++++++++++ tests/repackage/test_warc_filter_pipeline.py | 7 ++++--- tests/unit/test_aws.py | 3 ++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ce0ce91..dec5bf5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,12 @@ except ImportError: # pragma: no cover - exercised in minimal installs _HAS_DUCKDB = False +try: + import aioboto3 # noqa: F401 + _HAS_AIOBOTO3 = True +except ImportError: # pragma: no cover - exercised in minimal installs + _HAS_AIOBOTO3 = False + import functools from typing import Dict, Optional import requests @@ -175,6 +181,13 @@ def requires_duckdb(func): )(func) +def requires_aioboto3(func): + """Pytest decorator that skips a test if the optional aioboto3 dependency is missing (no AWS access needed).""" + return pytest.mark.skipif( + not _HAS_AIOBOTO3, reason='aioboto3 is not installed; install cdx_toolkit[s3] to enable these tests.' + )(func) + + @pytest.fixture def s3_tmpdir(): """S3 equivalent of tmpdir - provides a temporary S3 path and handles cleanup.""" diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index a0dedc0..3292063 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -20,6 +20,7 @@ from cdx_toolkit.repackage.records_source import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter from tests.fake_warc_server import FakeWarcServer +from tests.conftest import requires_aioboto3 def make_server(): @@ -248,9 +249,9 @@ def mixed_scheme_captures(): dict(hf, filename='hf://buckets/commoncrawl/commoncrawl/' + hf['filename']), relative] +@requires_aioboto3 @pytest.mark.parametrize('processes', [1, 2]) def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): - pytest.importorskip('aioboto3') source = RecordsSource(mixed_scheme_captures(), 'https://data.example.org') with patch('aioboto3.Session') as session, \ patch('cdx_toolkit.repackage.warc_filter.make_hf_reader', fake_make_hf_reader), \ @@ -271,8 +272,8 @@ def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): sorted(c['url'] for c in CAPTURES[:3]) +@requires_aioboto3 def test_s3_output_with_metadata_records(tmp_path): - pytest.importorskip('aioboto3') metadata = tmp_path / 'urls.txt' metadata.write_text('https://example.com/\n') s3 = FakeS3Client() @@ -315,8 +316,8 @@ async def read(self): asyncio.run(job.ranged_get_bytes(1, 0)) +@requires_aioboto3 def test_no_s3_client_without_s3(tmp_path): - pytest.importorskip('aioboto3') with patch('aioboto3.Session') as session, \ patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py index c133dec..029d561 100644 --- a/tests/unit/test_aws.py +++ b/tests/unit/test_aws.py @@ -4,6 +4,7 @@ import pytest from cdx_toolkit import aws +from tests.conftest import requires_aioboto3 FULL_DEFAULTS = { 'region_name': 'us-east-1', @@ -127,8 +128,8 @@ def test_s3fs_uses_the_defaults(no_user_config, monkeypatch): assert fs.max_concurrency == 2 +@requires_aioboto3 def test_warc_fetcher_s3_clients(no_user_config): - pytest.importorskip('aioboto3') from cdx_toolkit.repackage.records_source import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter From 92ebc03706d811eff0082bb2387bf57875c982d5 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 19:46:13 +0200 Subject: [PATCH 23/91] test: no remote requests by default: mock data.commoncrawl.org, S3 (moto) and DuckDB (local index) - the WARC records the repackage tests download are recorded in tests/data/repackage/warc_records.jsonl and served for data.commoncrawl.org range requests (responses; DISABLE_MOCK_RESPONSES=1 still uses the real server) - the S3 tests run against a local moto server with s3://commoncrawl holding the same records (configured with environment variables, so worker processes use it too); with CDXT_LIVE_TESTS=1 they run against AWS - the DuckDB end-to-end tests query a local columnar index built from the same captures (local_url_index fixture); the tests of Common Crawl's index on S3 and of Athena only run with CDXT_LIVE_TESTS=1, and the AWS access checks no longer run at collection time - the two remaining live requests of main (example.com, a non-existent host) are mocked - CI sets CDXT_DISABLE_ATHENA_TESTS (it set DISABLE_ATHENA_TESTS, which no test read) - moto is in the all extra (not in test: it would install boto3 for the minimal tests) --- .github/workflows/ci.yaml | 4 +- CONTRIBUTING.md | 10 +- requirements.txt | 1 + setup.py | 4 +- tests/conftest.py | 218 ++++++++++++++++++- tests/data/repackage/warc_records.jsonl | 10 + tests/repackage/test_repackage_cli.py | 50 +++-- tests/repackage/test_sql_index_e2e.py | 131 +++++++++++ tests/repackage/test_sql_index_gated.py | 81 ------- tests/repackage/test_warc_filter_pipeline.py | 17 +- tests/repackage/test_warc_from_fs.py | 7 +- tests/test_cli.py | 11 +- tests/unit/test_myrequests.py | 3 + tests/unit/test_warc.py | 6 +- 14 files changed, 419 insertions(+), 134 deletions(-) create mode 100644 tests/data/repackage/warc_records.jsonl create mode 100644 tests/repackage/test_sql_index_e2e.py delete mode 100644 tests/repackage/test_sql_index_gated.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3755861..29aab8e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,7 +93,7 @@ jobs: core.exportVariable('CDXT_CC_INDEX_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01') - core.exportVariable('DISABLE_ATHENA_TESTS', '1') + core.exportVariable('CDXT_DISABLE_ATHENA_TESTS', '1') core.exportVariable('LOGLEVEL', 'DEBUG') - name: Lint code @@ -152,7 +152,7 @@ jobs: core.exportVariable('CDXT_CC_INDEX_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01') - core.exportVariable('DISABLE_ATHENA_TESTS', '1') + core.exportVariable('CDXT_DISABLE_ATHENA_TESTS', '1') core.exportVariable('LOGLEVEL', 'DEBUG') - name: test minimal diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7f991b..3bdd738 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,12 +34,20 @@ make test make unit ``` -By default, all remote requests are mocked. To change this behaviour and actually call remote APIs (if you run this from a whitelisted IP address), the following environment variable can be set: +By default, all remote requests are mocked: the CDX API and HTTP downloads are served from recorded data in `tests/data`, and S3 is a local [moto](https://github.com/getmoto/moto) server (installed with `.[all]`; without it, the S3 tests are skipped). To change this behaviour and actually call remote APIs (if you run this from a whitelisted IP address), the following environment variable can be set: ```bash export DISABLE_MOCK_RESPONSES=1 ``` +The S3 tests run against AWS, and the tests that need AWS and cannot be mocked (Athena, DuckDB queries of Common Crawl's index on S3) run at all, only with AWS credentials and: + +```bash +export CDXT_LIVE_TESTS=1 +``` + +The WARC records that the `cdxt repackage` tests download (from `data.commoncrawl.org` or `s3://commoncrawl`) are recorded in `tests/data/repackage/warc_records.jsonl`, and DuckDB queries a small local index built from the same captures (the `local_url_index` fixture in `tests/conftest.py`). + If the remote APIs change, new mock data can be semi-automatically collected by setting another environment variable, running corresponding unit tests, and overwriting existing mock data in `tests/data/mock_responses`: ```bash diff --git a/requirements.txt b/requirements.txt index 2901643..a1150de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ pytest-sugar>=0.9.4 coveralls>=3.1.0 flake8>=7.3.0 responses==0.25.8 +moto[server]>=5 pre-commit>=4.3.0 # packaging diff --git a/setup.py b/setup.py index ce6eb86..39e9b9e 100755 --- a/setup.py +++ b/setup.py @@ -11,6 +11,8 @@ requires = ['requests', 'warcio', 'fsspec', 'aiofiles', 'url-is-in>=0.1.1'] test_requirements = ['pytest', 'pytest-cov', 'flake8', 'responses'] +# S3 mocked with a local server in the S3 tests (not in `test`: it installs boto3 for the minimal tests) +test_s3_requirements = ['moto[server]'] # S3 reads/writes (index files, WARCs) optional_s3_requirements = ['fsspec[s3]', 'botocore', 'aioboto3'] # `cdxt index athena://...` @@ -36,7 +38,7 @@ 'package': package_requirements, 'dev': package_requirements, 'all': ( - test_requirements + package_requirements + dev_requirements + test_requirements + test_s3_requirements + package_requirements + dev_requirements + optional_s3_requirements + optional_athena_requirements + optional_duckdb_requirements + optional_hf_requirements + optional_parquet_requirements ), diff --git a/tests/conftest.py b/tests/conftest.py index dec5bf5..3cefa0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,19 @@ +"""Shared test fixtures. + +By default, the tests make no remote requests: +- the CDX API and HTTP requests (data.commoncrawl.org) are mocked with `responses` and recorded data + (DISABLE_MOCK_RESPONSES=1 sends them to the real servers, see mock_response_from_jsonl) +- S3 is a local moto server (see the s3_env fixture and requires_aws_s3) + +CDXT_LIVE_TESTS=1 runs the S3 tests against AWS instead, and enables the tests that need AWS and +cannot be mocked (Athena, DuckDB over s3://commoncrawl; see requires_live_aws). +""" +import gzip import json import os +import re import uuid +import urllib.request from pathlib import Path import pytest @@ -31,7 +44,7 @@ _HAS_FSSPEC = False try: - import duckdb # noqa: F401 + import duckdb _HAS_DUCKDB = True except ImportError: # pragma: no cover - exercised in minimal installs _HAS_DUCKDB = False @@ -42,6 +55,13 @@ except ImportError: # pragma: no cover - exercised in minimal installs _HAS_AIOBOTO3 = False +try: + import moto.server # noqa: F401 + import s3fs as _s3fs # noqa: F401 + _HAS_MOTO = _HAS_AIOBOTO3 +except ImportError: # pragma: no cover - exercised in minimal installs + _HAS_MOTO = False + import functools from typing import Dict, Optional import requests @@ -58,6 +78,7 @@ TEST_ATHENA_DATABASE = 'ccindex' DISABLE_ATHENA_TESTS = bool(os.environ.get('CDXT_DISABLE_ATHENA_TESTS', False)) DISABLE_S3_TESTS = bool(os.environ.get('CDXT_DISABLE_S3_TESTS', False)) +LIVE_TESTS = bool(os.environ.get('CDXT_LIVE_TESTS', False)) # Cache for AWS S3/Athena access check to avoid repeated network calls _aws_s3_access_cache = None @@ -104,13 +125,13 @@ def check_aws_s3_access(): return _aws_s3_access_cache -def requires_aws_s3(func): - """Pytest decorator that skips test if AWS S3 access is not available or disabled.""" +def requires_live_aws(func): + """Pytest decorator for tests that need AWS itself (not mockable, e.g. DuckDB over s3://commoncrawl): + they only run with CDXT_LIVE_TESTS=1 and AWS S3 access.""" + if not LIVE_TESTS: + return pytest.mark.skip(reason='needs AWS; set CDXT_LIVE_TESTS=1 to run it')(func) if not _HAS_BOTOCORE or not _HAS_FSSPEC: - return pytest.mark.skipif( - True, reason='S3 dependencies are not installed; install warcio[s3] to enable S3 tests.' - )(func) - + return pytest.mark.skip(reason='S3 dependencies are not installed; install cdx_toolkit[s3]')(func) return pytest.mark.skipif(DISABLE_S3_TESTS, reason='AWS S3 access is disabled via environment variable.')( pytest.mark.skipif( not check_aws_s3_access(), reason='AWS S3 access not available (no credentials or permissions)' @@ -118,6 +139,16 @@ def requires_aws_s3(func): ) +def requires_aws_s3(func): + """Pytest decorator for S3 tests: they run against a local moto server (the s3_env fixture), or against + AWS with CDXT_LIVE_TESTS=1.""" + if LIVE_TESTS: + return pytest.mark.usefixtures('s3_env')(requires_live_aws(func)) + if not _HAS_MOTO: + return pytest.mark.skip(reason='moto, s3fs and aioboto3 are needed to mock S3; install cdx_toolkit[all]')(func) + return pytest.mark.usefixtures('s3_env')(func) + + def check_aws_athena_query_execution_access(): """Check if AWS Athena StartQueryExecution permission is available.""" try: @@ -161,7 +192,9 @@ def check_aws_athena_access(): def requires_aws_athena(func): - """Pytest decorator that skips test if AWS Athena access is not available.""" + """Pytest decorator that skips test if AWS Athena access is not available (needs CDXT_LIVE_TESTS=1).""" + if not LIVE_TESTS: + return pytest.mark.skip(reason='needs AWS Athena; set CDXT_LIVE_TESTS=1 to run it')(func) if not _HAS_BOTO3: return pytest.mark.skipif( True, reason='S3 dependencies are not installed; install cdx_toolkit[s3] to enable Athena tests.' @@ -188,8 +221,175 @@ def requires_aioboto3(func): )(func) +# The WARC records of the first 10 captures of tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.gz, +# recorded from data.commoncrawl.org: {(filename, offset, length): bytes} +WARC_RECORDS = {} +with open(TEST_DATA_PATH / 'repackage' / 'warc_records.jsonl') as _f: + for _line in _f: + _r = json.loads(_line) + WARC_RECORDS[(_r['filename'], _r['offset'], _r['length'])] = base64.b64decode(_r['data']) + + +def _data_commoncrawl_range(request): + """responses callback: a range request to data.commoncrawl.org, served from WARC_RECORDS.""" + filename = request.url.split('://', 1)[1].split('/', 1)[1] + start, end = (int(n) for n in request.headers['Range'][len('bytes='):].split('-')) + data = WARC_RECORDS.get((filename, start, end - start + 1)) + if data is None: + return 404, {}, f'no recorded WARC record for {filename} bytes {start}-{end} (see tests/conftest.py)' + return 206, {'Content-Range': f'bytes {start}-{end}/*'}, data + + +@pytest.fixture +def mock_data_commoncrawl(): + """WARC record downloads from data.commoncrawl.org are served from recorded data (unless + DISABLE_MOCK_RESPONSES=1); other requests made with `requests` fail, unless the test adds mock data (e.g. + with mock_response_from_jsonl).""" + if os.environ.get('DISABLE_MOCK_RESPONSES'): + yield None + return + responses.start() + try: + responses.add_callback(responses.GET, re.compile(r'https://data\.commoncrawl\.org/.*'), + callback=_data_commoncrawl_range) + yield responses.mock + finally: + responses.stop() + responses.reset() + + +# crawl and host of the captures of WARC_RECORDS +LOCAL_INDEX_CRAWL = 'CC-MAIN-2024-30' +LOCAL_INDEX_HOST = 'bibliotheque.missiondefrance.fr' + + +@pytest.fixture +def local_url_index(tmp_path): + """A columnar URL index like Common Crawl's (/crawl=.../subset=warc/*.parquet) with the captures of + WARC_RECORDS, and captures that are not selected by the tests: of another host, and of another crawl.""" + from cdx_toolkit.record_io import parse_cdxj_line + + with gzip.open(TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz', 'rt') as f: + captures = [parse_cdxj_line(f.readline()) for _ in range(len(WARC_RECORDS))] + rows = {LOCAL_INDEX_CRAWL: captures + [dict(captures[0], url='https://example.com/', urlkey='com,example)/')], + 'CC-MAIN-2024-26': [captures[0]]} + + base = tmp_path / 'url-index' + con = duckdb.connect() + for crawl, crawl_rows in rows.items(): + path = base / f'crawl={crawl}' / 'subset=warc' + path.mkdir(parents=True) + con.execute('''CREATE OR REPLACE TABLE idx ( + url_surtkey VARCHAR, url VARCHAR, url_host_name VARCHAR, url_host_registered_domain VARCHAR, + url_host_tld VARCHAR, fetch_time TIMESTAMP, fetch_status SMALLINT, content_mime_type VARCHAR, + content_mime_detected VARCHAR, content_digest VARCHAR, content_languages VARCHAR, content_charset VARCHAR, + fetch_redirect VARCHAR, warc_filename VARCHAR, warc_record_offset INTEGER, warc_record_length INTEGER)''') + for c in crawl_rows: + host = c['url'].split('/')[2] + values = "?, ?, ?, ?, ?, strptime(?, '%Y%m%d%H%M%S'), ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?" + con.execute(f'INSERT INTO idx VALUES ({values})', [ + c['urlkey'], c['url'], host, '.'.join(host.split('.')[-2:]), host.split('.')[-1], c['timestamp'], + int(c['status']), c['mime'], c['mime-detected'], c['digest'], c['languages'], c.get('charset'), + c['filename'], int(c['offset']), int(c['length']), + ]) + con.execute(f"COPY idx TO '{path}/part-00000.parquet' (FORMAT parquet)") + return base + + +class InlineExecutor: + """ProcessPoolExecutor stand-in that runs the workers in this process (mocks apply, coverage is measured).""" + + def __init__(self, max_workers=None, mp_context=None, initializer=None, initargs=()): + if initializer: + initializer(*initargs) + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def map(self, fn, items): + return [fn(item) for item in items] + + +class S3Env: + """S3 for a test: a local moto server with the buckets commoncrawl (the WARC files of WARC_RECORDS) and + TEST_S3_BUCKET, or AWS itself (CDXT_LIVE_TESTS=1).""" + + def __init__(self, offsets=None): + # (filename, offset) -> offset in the mocked WARC file, which holds only the recorded records + self.offsets = offsets + + def capture(self, capture): + """The capture with the offset of its record in s3://commoncrawl (the mocked files are short).""" + if self.offsets is None: + return capture + offset = self.offsets[(capture['filename'], int(capture['offset']))] + return dict(capture, offset=type(capture['offset'])(offset)) + + +@pytest.fixture(scope='session') +def moto_server(): + import logging + from moto.server import ThreadedMotoServer + + logging.getLogger('werkzeug').setLevel(logging.WARNING) + server = ThreadedMotoServer(ip_address='127.0.0.1', port=0, verbose=False) + server.start() + host, port = server.get_host_and_port() + yield f'http://{host}:{port}' + server.stop() + + +@pytest.fixture +def s3_env(request, monkeypatch, tmp_path): + """S3 for a test, see S3Env. The moto server is configured with environment variables only, so that also + worker processes (cdxt repackage --processes) use it.""" + if LIVE_TESTS: + yield S3Env() + return + + import fsspec.config + import s3fs + from cdx_toolkit import aws + + endpoint = request.getfixturevalue('moto_server') + # path-style addressing: the endpoint is an IP address (an explicit S3 setting of the AWS config file wins + # over cdxt's defaults) + config_file = tmp_path / 'aws-config' + config_file.write_text('[default]\nregion = us-east-1\ns3 =\n addressing_style = path\n') + for name in ('AWS_PROFILE', 'AWS_SESSION_TOKEN', 'AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_MAX_ATTEMPTS', + 'AWS_RETRY_MODE', 'CDXT_AWS_DEFAULTS'): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv('AWS_ENDPOINT_URL_S3', endpoint) + monkeypatch.setenv('AWS_CONFIG_FILE', str(config_file)) + monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', str(tmp_path / 'aws-credentials')) + monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'testing') + monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'testing') + monkeypatch.setitem(fsspec.config.conf, 's3', {}) + aws.user_aws_config.cache_clear() + s3fs.S3FileSystem.clear_instance_cache() + + urllib.request.urlopen(urllib.request.Request(f'{endpoint}/moto-api/reset', method='POST')) + client = boto3.client('s3', endpoint_url=endpoint, region_name='us-east-1') + client.create_bucket(Bucket='commoncrawl') + client.create_bucket(Bucket=TEST_S3_BUCKET) + offsets, files = {}, {} + for (filename, offset, _), data in sorted(WARC_RECORDS.items()): + offsets[(filename, offset)] = len(files.get(filename, b'')) + files[filename] = files.get(filename, b'') + data + for filename, data in files.items(): + client.put_object(Bucket='commoncrawl', Key=filename, Body=data) + + yield S3Env(offsets) + + aws.user_aws_config.cache_clear() + s3fs.S3FileSystem.clear_instance_cache() + + @pytest.fixture -def s3_tmpdir(): +def s3_tmpdir(s3_env): """S3 equivalent of tmpdir - provides a temporary S3 path and handles cleanup.""" bucket_name = TEST_S3_BUCKET diff --git a/tests/data/repackage/warc_records.jsonl b/tests/data/repackage/warc_records.jsonl new file mode 100644 index 0000000..ef6822e --- /dev/null +++ b/tests/data/repackage/warc_records.jsonl @@ -0,0 +1,10 @@ +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514759.37/warc/CC-MAIN-20240716142214-20240716172214-00337.warc.gz", "offset": 111440525, "length": 9754, "data": "H4sIAAAAAAAAAO2d23LbSNKgr62IfodqTtiQt80DDiQAydQELcndcus0ltzdMx0djAJQpGCDABsHye6JjtgH2Yu92r8d/xNs7KXeZO/2LTazCgBJERRJkZT1W+wZSxRQh6ysrMz8qkDy59bb3apcqX2z8TO8Kp9/6rMtErKoH/gRSy/u0RguKjVFK9f0stw4l+tbqrxVr/8rLfCW2UHolA/2tsjLJPS3ksR1tmSVqpbhyGXdZEpZ6zBWNjSblRVN1nTasSxVsXe+2dgN/Jj5cfmQ+d34YovodUOpDy4LgWi/77k2jd3Ar17EcX+b9KJuDLeaN0T9mYa263eCG7LUDaZYms7KOlOdsqbbjbKhm1qZmppZa9TqjqxZO2kT0LOdhCHvPBgZkEZrHWaYoATTKmsNyyhbtYZatmt1jVHbUW3DzBo5OC23HAeEi7aIXDMriqpV5EajouSKpmGXxeV3bw+2CI4o2qpWLdfy3CC+YL8nrNJzowiG67BOSH2bVTph1YW/Plb6F/2/e5de00o8j8Wu33bcqO/RT89cp6nKZtr+aRjEgR14onWYYXnsRuxFcF3Nhu32L1hYPktcnOzzw7P27g8t+L9Sa5+eHP5TVmv19hn8WW9kDdFPXkCd8p7bZVG8RaILKm/t1X9UX7/7ee9t6+To6PzNrrJ7/vNP+qHSevX64E391Wla95UX2B9Ga/7w+vxVXd87OfjX93unZ292f9SP/9E61xqnx/Wjkx/e7mWKdWBi3I7LnFyAcRP5eBH3vO8+9rxvNr7Z+OH8/BTHDwZcIyc/frMhzPk8YS+I3CBvEo+bNkmNWiPfH53ftL+YfYyr2Og2sS9oGLG4mcSdsvHNxi/l3ZBeeaC5c5inqAMv9n07cFy/uwVlE/8Dc3hrPrNRti3ygbF+mXruJdjsTzT8tEVats36cV4PilP7gqEhxiFOVD+I4jJcsT80ay9IP2RDf0y69c3G/se+Cwa4Rc5o/ILUZPKG+qiDGqnVtvj/xUjPwAx3g+CDC+M8vegffXqFZlg+OW3t7r1q2tRmDZz7YePcJkw03uRtK0b5TeKXQYt16GCrnmlxmxzRj+VWlzXVmvjvRm8/nJ7tn50d7DVlymRTUy1mylbDgu5qTLYUxbQMajc6HWub9Gl80azeqN+zTvrULh+efH9w3HQYLAfmDGQ7v0hw3GUYd1k29Xzc8qhsN4VKG0XJjltH+830wuRSIL9hwMprNEyzXlxsr3XeetU6219EyEMKU30EFoLGP9V+z1h4ycItEijRlRvbF+XT4IqFrwK0l0t12G4zSx+YrRUWuGXN1A1cThsvv9072T3/5+k+wQVBTt+9OjzYJaVytfqzulut7p3vkV9+OD86JLDmqtX949IGgf9K6IbAx11dXVWu1EoQdqvnb8VClWWslL2uOLFT2tl4yRuHJexHzYK6smmaokYJC2151O82S52wlK9PiS9QCVti1NnZePKyx2LKvWEZbNi9bJbsdIwYSKCi+KtZmrTYS6SaN+PTHmuWaBJfBOFQ1dOjV+T7MEj6vOxo4Q/s0xVEyWioeOrjicOI8PIvyPAyG+/QYZEdun10JEPNvErrPGPdkF6ybahKjgZNv+ZNF0gUBlYQD8tDPS/t89ty+WCfsJhE0Abzwa1hS3bQg3XoWq7nxv+vXC5SqvRL+V0LbEoU9JiUty4d7Df3nS6Tbozp0mVX/SCMh+S4ch1Y7Q67dCFf4H+8IK7vxi71ypFNPdaUX5Ae/ej2kl52AWzmyZMn0HTsxh7b2aUx9YIuaIL5xHO7foFKXlZF2Y0n+L+Xnut/gNTHa0pR/Mlj0QVjsUR4liFxm7CjSCIXIes0pUpVlKmCSnoQccSvCpT4u6wrtYapaI2aJHS5SLsdUEmZXrEo6LHltx70mV+OIGwtv+mQp4RZbrLE9vs9S2v3nA505GFUHZVdN5bSdhdWAgtd1u5fQLa0ki6yFytpO3T9eBUNx7S7EmVcurgyfZZErLh58EcEbrTR42ICKBI9Aj5ouOuzH07enu++OycHuyfHWW9uj0KaWY0gr612KLgUWKfwg7sh4U2FmMLvv6eXVFwtkSi0myXXt70E/O7QnaoT0m4bEu8w6FfeR6Wdl1VxY4EmL6jveGypTfaDfnJ7Y9KNxlAnhDxxO2TzWyewkx6ougKYsu8xfBm9+rTr0Sg6Br/9/N+kWoX0MwmJy9DzTilPmgRLdRKfp8GbftBr23jz+b/x+pNLCg3FrBc1fXZFWmFIP20+385v2UECgaEmLnSCkGzyCnCFuC+Luz6nXex4U/pv0vOKx9MYKPzdd8/JvwmX5Qkf6AyVUa5N93nFHgymSQYDIGIE0BzK/ysX9bvvfoMRz972tpAIM6U/xSvxK2Qx8KfQDB88v47/QPuvA6FMQkkSQ0yOXAjTfEo8iWE2B4YP0S4i794eRhgHBzO9MZgKwouyNhTadCBqiglB9YL4u5AAQUI7PBbMiagdM3797xOub41ehxyKS+92NocbbZLSu/PXkFmlRpCOVggEaIxpROBDI0Iw3sSfzIvYaHHIAPpsuIhQ0cDqN+Z0T4VrPngfVB33vRtXwQX14Gac+OAI0l8V3gD4qqkLTGj2fQAZdsftgmZxMH3U0Yl/CDwLtBlCbg8XAZIhtdkiUicsd0IJL7nRHrMSyNE7FNSAV8BlnnrU9d9ECJdZVYCOD+j2tsivXFeYakFD4HBf0YjxpnjzHEmlSqXK/y+h4n7Dmw7rY10JqJq76eoRiyJs8CwOQibxMiCcZ0E/W/miHikk5vTK9Z3gqgJN7MGYcbR/8jlKL1R6aRW4gwt/pIXNfyehB9JV6Xsq9j16gZNAsod/PwPhWbeZ1ZdeEAeAyo4PIE/c4tr5k1sDDgmkhV4H9nD7HHGPeqsVZD/AuSJgZO0uo1nQS97yrALvoEkFMOXt6CK4akeBDYly22cx8MYHAj5yZv8vZCySMHJ74GHaEaOhfXFDupkaLR44kkM7RNiJ4putgmIFum28tALnEwl83O5BOuAm5bAOTbz4LKZxEjUlRC++lXL9V0T6CRCRLW2XCHGhBigV1VOaYXXmXrEDvRwHkGmw1HNvug43aVR24jX9xPO2uR+2acS9rM9LR4RHBReHtJH50Z5oqTgkvPp04GxKzJO+c53vpN0L13Ok56Jp0WSWRW9wB/pt3tzz2VoWjYgmuQPOqueOPvEi3kB6Y0K4SjzRxkj8JRiAoX4aYYkIscI/Q2dw51f3N2ywFcehayUxNOTC+ND5S4Hf9Vgctd3IctrpnH2XqjkXDSQTrYjg/8QKGf0gXv+Ze3uSRwWs5Lm36UP0yjvlvWWR6VvPfU5mrj4kbtZAVp/L7LkVcOpQ6xhi2XYatNOIlaDl/DlkbHzdXl3QePP3JHABdh2RVYi9G35RKAVsjRFp/5fTw/bhya5EtrgeJsrquJdt9rHvtcHXS+Q7bLfCA2AlNSkQVLJwM1banq2lia34EKilbe5vb2kmVd5wS0NZFZGGLKHt+pBOuJfztTo80oktZ+1ujNrTQLnLUaxQyYJ6nWl25lHryOCXptLBZN1QauqmU4WiJTcJ/qzEwSFuSe6C0oczfb54Cp3v9kaWsydennTn3EC4n5jgujw3dV0FzovXLnJffE2LnH6aC2tL3+GYhnxXRgNQc7aZmGYkgx5ut5GRDPnGGAaSfDsQHZleIs+ekduLATnPUixKut0Zilk0+gAZ+EBbM6hryB/ktSap7JZpq4QM9GazzMalF6hf6fntCziLNoPffz5J3dKfoxb/J7r2pSRHMYVxR6B6TH/nSQhnaHRp7fXcyB7LB0GhmHhJNMbKqNam9Ef5AM8vt+RarYalocxQUb4ZDxjDwtJOfq0HFwZ/SvhnG1NCFkIDF+pOtvF9/RdueTuMeLRolxdKiu5GGm5fBGHU7gRBjH2mc7uAInBrqp3tuQypA9rMunXxQK+dIks6MG7ukKIqpUzIEVElHxZelwIOIWlEUN/v7rQIpBaxK/bkWRwz+BtQ/zJIIuT/S/YH2YL+ReGNlzjljIhNdQnU/xSki0P456S9i064Pwp80IkbgtWB+Iz5Etl5SQUYlyo3TqD5X6XRNuB1nCbC0EvUp/7OWxYHkPWwkNAEjyh7uE1x/dmG2SH0ktkEeBxGEvru9V/hII+uVCowBmzgZZXCv9hB1EK5uQ2zTFG4NdhhYQ9TK24Ef8tVhvuDufYlaBt8RRtaBAGkTGzW68efpGGtpzPCbeQqBALmNjlZidx20vtpozYNQ5iMiHnt7Jg+XweSsLVsBtKrvOEt7em2lOtbmn7eL9fkOlRwe11C8ZAD1ojrOHjqkhmq2PwErV+6IJBc6ftdqTrQ6JANpM51VCBZrciaYtR18XN+8VRpR2k0SJn85F7C9NgSgB6BNCt0I1izYAf/k9jh9WcWvrRCUt2JWB88N/jScmDHAfwmSk1W7k9cDcXVQdw96kfcMCFS2UmaURAnIT18JuMFB74IryLxOdL+JfW7rgc3WjDrPcuDIZHvrz/7159D6qWj4CP0g0sxQAeWAH91z0Os4xANGOLJ90dAzCAcTA31Y9wg5BwOPkCI+p76l7BUyp3rz5chLloQVC0SNH1ehtt6e2D7S5VclU0U3ATBDynpICV5YDzB9f8m4O198P0YbZjvgC+MhPwUpPbKsI4nCr4aDTdAUL0Ggr52ffRsoGCWajRx/TL8wGplGlz/n/ieRdNRNBlE270AP+yTDnUjmPsXuDfqhuyWRXhfQqo1BWVUQMbzEKIr6iokt62dUckW9aeNWfypD3nBmC8Vbh0j1IZ4IaIUZh/Y4GgLeAyF66UtNz7KjUrX7eDuMdmYIOzoRhCKqoGrOoTYDzqgyUeyG/R6iQ85yPXnCCXaqIDOcL28rFo7z3wr6m8PxXFIm9t+0uPNI+LB6za02VBKJOYPpjVLbcuj/ofSQPZSLnvXCyzWDiB+dxkqoURAW1DlsLULHbZxSqAdPFofvZbpCjSyka/hZ1Q8tECCZ0n2SqznZ3+rqeb2zUV9hMd/Ir9zaMzE0LYIES9Gl/yGqHJKwwT83I2iNbla06pYjGwI6xlJLnhNnLy968+QU6EZ8hgg0knMS7JEARRIfafleaWBqQ3ywq38Ns/jN+Xn22KXdnhDtU09bzO9wZH7MnAjMM2Ru6lhpksPXrb7XhLhv5vmKbrEasKwrCCE/Kop1SSeCInbcFdKJ0RkWSNjgj9s9hYyJi8+40KUMisqKG0Hnkf7EZuog6ECm8/Hx9ELwFHyHzdHklWcOJaswNhoMmOHcBS38ZFQbuwikDzDJUV96n2K3OiZIIrmwPTzKlluW0IhBbJmhi213geAlCHGT8hfUE80BosEh38Z4KJM08yd2cpxyTnGA+XHLPUl0zar8T8w13aaMY/b1K+33k03PX5ryoaiKtup/8opLsXbnF6Yx8ud8l3NXDWi9bLY6yxlgg8ZR9a9A9jt4tqKSrDCd4HoP2RrB8+kcAecty+94KdYz7fJ+DrZlESRiUtlUICke638KGi7xE+2CQHZCtaQlDk4mH4wMm4pftCDl028/SzgT1z5zGvKpfSpJbe3D+gzpJaDXjd3ehCegGW9UmaspVqJXPAV1Syp+IRVgVPlAQENLw8IJe5VpRaQnu2lD6Ey4vb6odt7xqgNZrUt5eZ4a6nqxHkpo/XTEsFHGro4O3hD/CmeNxDFxFDFcw9woc2HXBr1AhGexCD8gZJGnUqKdM10ckimf7Hy2nwI0s4e42moh6HP7UDSGTL+SJwdBpCaICkA3ndCcPqAihDnCjxR1qJ4MA8IlFRJy7NYGJPNo25YeU7enrzbP09rkqKqQHtugC5qB9KjCQH2BcFEdHM02Dx/WXWLfKn0L9U0jHyi7Phj+xJsAq5WDKOs1GraM9rrb4eduH1JvXanFzfR9zxVW0HHfaq8hgvw+gO7hJ+9+CP8fI84Tb2sWqXL/JA1qbCA/CoVHQqtPlVq3lNFF5p9uqs+bZlcu0+VXbglNAwv+J0aFqbwI9e1qJC3/F60LJQD5UA98DNXkCgd5cXdKEpYEzSWX8HA3cy1B3VRf/ldhpsZTbVs9p+hUsDwBmNKcH+hySdx6CLfsGiKicZRKAZO9lPFlPIJGfmV+wIe5FO/J/ajChweP5G76e9scVFkm6UeRA7I7K0gjoPeVqP/cTvN1LZwCxGPIUfcGCwEsIfgA4ME07QaMHrLanQcw6nbHVOtdwyLWSZtNMA2NImnObHbw+WjK7KsqSZi6pQNiZeJx+NjwRFbughT6yw6gxOPAT3BR6uGGimsXrCfPcgA/pZZfemUl0DHbwvHPzjxksQTxOD2J/puUVtEd89NZ2lUNpRjsmT51nGRbAdnr/aKJcM7t8mF90el2hDP8DwRBJB4AgUGu0pSGFwNNj8Htsa3+Ef0m3JMZkc8N9iWRj1LJ/CdcrrupZ1f01e/pVae5mtpPhuHWa0497jISynnhG73Is51Z3XL3ZB9utEdWOHvbfsC1p20c+6iY9zK15Wgr9HyqfIXdO+D1Hx0CJ/6QDGLjQECHOeI9BAhA4SbgxoOqShXsUDg+ViCMWcBiVqiDTIuQ263NzdhRagDx8I4wMqqqkpkpzDo0Z0XRPQwaQyfwD0tNgDfx50vUKqYfP6A0SS9CnYvlsTv86d6FpJFLAfc18LGNqPnBXrFMDNJBnxTQbKgEIeijfGVgocT1/9B3Wiod8GcFmQeSO1k4pwX7gaQG4sP85hs7U3cKMj318kt3U3ajrvRI9/9SzvE/KggQRp0VxFGURa/0lngnmoj8448Qg/7xyEff8M78ih7B+c4r08iv476gt8gv1zuyhRyjWqHZ0cF11FrldRoptjfyKi3yMD4HrfJZQdJIwdKY3tBArRHwnXBJs1o9jFeoDDsjzL5aBMj9/ITqqE89T63B9QZtwfUFW8PqNO3B9Rp2wPq/W8PqI9ve0C92/aAesv2APNdiM8XIejPhahOE/KG9gN/xo2AIxq6rHziuJATnJ4cnx/sv31kGwGoP8T9i1AwOmqR8zv84Jq8X8Q3y7I+kfHTGRqj/KFZXAbdqyume60mG7JZpxpT66ru2Jap1ZlGTctWLNXRV0P36mJ0r66O7otc7xx0XyzZEui+SK6V0b36ddL9FO+85vi5OF4HJ7JTGLLWGD8uAw8la47/0lCVTcOMID/uCBcA+QnuZyFkn2URzkPsXD9rZF/IupbN7OrizK4+TGbXZmR2bcXMrk1ndm0as2v3z+za42N27W7Mrk1i9l2xj8ofd0l88ewswQ8cIXsuSx4Tfe9mR+yoC/iVIHhzfcDvSFxDndwvgdfr5YaaI/gyeFpbLU9rtFZT61rdoLZlwIRa1JZNy2goHdroMKquhqe1xXhaWx1PF7nFOXi6WLIl8HSRXCvjae0r5OmZPOd/GaZ+fEAqPOuaSL80M+TzMCOSjvuSuyLp7St4DEvnwUgxqDVHLmYTywZJbXGQ1B4mSNZnBMn6ikGyPh0k69NAsn7/IFl/fCBZvxtI1ieB5E8uA8dJExIxO8CddfClPQp1fXwb0cwHwF6QkMN98qr19uwxwSfXXnbQm2oQURS5sodPeoMe8XnvwL1f/GyoZV2eeAJ8uA+1cKoKzoBhIpdBq/XV0qrFHIVaTFM0o2bQuqw1OpqpOszRZbXTsFf0bHd9MVqtr45Wi3zvHLRaLNkSaLVIrpXRav0rpNXZ3PN/FVp9ICfAiqbw06eRoLU+/B2XQYSRNWt/aa7K52FG1h73hHdl7dv9z0JHwFNW4TzYLvSzxvbFzGvZ2F5fHNvrDw/bFV3T9BmwnZdbIbbz9m/HdlHkFmzPCtwjtvMuHxe28yHPj+1iciZg+z8S/nlPNsNPFsreVZPuiMLficc/s4t63evPYVDuJVHi9Sjc+fuMRP+Gwe3zk3dnZ62D40f1tu5/AMrroFt8nBv5HPQ7/Ebu/JSZXxV6Ror3uumbv4e0je+YVl/fL/n34bKsTyL/fEbHyB8nfFHw5xa7WvB3WN0C2m8Yqt2xaF1nusEcatfkjlHX6voKwD9dhncF/9HqSwX/Sd57RvCfLNmC4D9JrpWAfzqKrw78F3bw6z2B+fYE6pBr7twMe9M3Be4LxftE1tcg/qVJKZ2FGTC82C/dFcPv7A0WI/Qpa6IIxVFDaxBfxLyWieGjMf5OGD7axIPCcGNGDDdWjOHGdAw3pmG4cf8Ybjw+DDfuhuHGlMewIXK3OqHLfTOYFH4w8aycnTiuTa//0yOvTt792Dr+vvWYWDt/dpu/WzpVIQdvocT7R2dFnoTO2fyMkzOfQi7CK28ZCG2sFqFlo2E4CqWWITtGw6pbYAGMQcZi2TXT0lfwpHe6hBZBaGN1CF3keedA6GLJloDQRXKtDKGNrxChpzvnNSPPw8iKyd+1WRSyHhInK/Kak788yPBZmJGTx53Pgo+GFyz5RUB4JsMvhmFFXsPwIja0bBg2Fodh42HCsDkjDJsrhmFzOgyb02DYvH8YNh8fDJt3g2FzEgy/DlyMyZD1vGeJz6L/+9//V2VGDD6lcQj2RM5ah63r/3H8mBgYtPZUMaKnipk9OC6093RfeWrUnrYalS9AwRMPkHGCsB9z93gMhNNJXAYCm6tFYMPRaUOrmTXDcExm2bLNaMcxbFPVaqqBxrAKBDYXQ2BzdQhc5G/nQOBiyZaAwEVyrQyBza8QgW9zyWv4nfOh8Rokt2OB6kGR7/qE+CFQyxwnxOM+567kW7TSF3w8e4q9TwDe9envQqazbOA1Fwde80ECb702G/DWa6sF3nptKvBCkduBlxe4X+Ct1x4d8NZrdwJemJwJwFv6Cb8XmuK3Qnv8WxdL5NsZgfcHFl5ef8ZvKfhl//Ax4e5TRUGtIb2C3vIvxEIIVvg3TMn3j7uqesv3YcH8jJEuzp7ofgmwW6+t+JO9bMWSbdupaUzRVaVh1B1dt4HxFbNh1mpsJbCLi2YB2B2qvmzYLfS1s8PuBMkWh91CuVYFuziKrw52b3PHa9idD3Yb+I06N4LUQ0JddeJ7o9eoe3+8wmdhNtQt8Dh3Rd2idb4Y6t5u7cWgq6pr0F3EcJYMukNx+a6gO9TEgwJdeUbQlVcMuvJ00JWnga58/6ArPz7Qle8GuvL0x5xB9mjm74XCd46Uhb8DSyCn+8c/PLYviR59wllo7wsArjnxy6DSOSl8PzCfOy6FDvO3DNSVV4u6VDcUXW4A7lp1TeswAz8eTK01LEsB5K2bq0FdeTHUlVeHukXedg7ULZZsCahbJNfKUFf+ClF3kkNeY+5cDzRrZvYGx8Iw9aCA11wD7wPgFnN24B33O4s/1SxW+0LPMs9s8xOw11xj7yLms2zslRfHXvlhYq8yI/YqK8ZeZTr2KtOwV7l/7FUeH/Yqd8NeZRL2vmEkSsA7UnSyPPsZ/oqQKfC7R333+i8I7Lsn796enxw8qs/FfsO/fAl0h1Tr9gTJsuwB5/v/OiaEYG3iKW82Q2MQjHPIBTAW/05kbmkr/k5kvWEzZhm6aZoN3XR0rWFbVGeqWlcc/mGKq8BfZTH8VVaHv0Vedw78LZZsCfhbJNfK8Ff5CvH3dse8huD5INhAICgIVw8Jf7X1ee8D4BdtjvPecb9zV/wtXu2LQfB0my/GX2196ruQ+Swbf5XF8Vd5mPirzoi/6orxV52Ov+o0/FXvH3/Vx4e/6t3wV52Ev0cU5swHNYBbmfV9vIAybh+SpPP9H39sPaoHm4/wOeZcX18Abyd/6DOfi/G366ZztQywVVf8fl2Lyg0DGVZxVJhIQ5Frikk1TVMcp15fzUdW1dXFwFZdHdgW+dM5wLZYsiWAbZFcKwNb9SsE2yKXu8bZOT+kCr9WZiwQPSiYXb9P9yHQyOzv0y3wNXeF2eEVvuBnUk2x8wkAu35/7kIms2yAVRcHWPVhAqw2I8BqKwZYbTrAatMAVrt/gNUeH8BqdwNYbRLAHvJ3h0AIjhjpCecCkbmH0zor0L4Orz+D8kPXJif/ah0f7z8mpD3M35pbg6r8G47xe40zlOWPNAtt3j/u1id+ULOYpzHcfR2Kzp30e5oQ4RYHX2214FunjuPIDV2znY5j6p1GR1dsu2Gxjq0aWoeuBny1xcBXWx34FvnhOcC3WLIlgG+RXCsDX+0rBN9ZXPUahOcCYQM/qXSnIIA9JBSurz+s+QFwTX32D2su8D53ReHb1vxCaDyD5RfDcX39ac0LGdGy4VhbHI61hwnHjRnhuLFiOG5Mh+PGNDhu3D8cNx4fHDfuBseNSXA84ZvdS+lXu5dmBOQb39/+eOCY648TLgdjrsPsAece5R9lpYAu8w+3un9Ebkx84PlwH27jhI1RMp/OZYBxY8UfamXQRoM5jskU2qkzGxa7DuyraLquGXWmrAaMG4uBcWN1YFzkg+cA42LJlgDGRXKtDIwbXyEYz+qm13A856c54+nZjeD1kMC4sX7g+QEwTWOOB57Hvc9dwXjaml/wc51vtfxiMG6sH3teyIiWDcaNxcG48TDBWJ8RjPUVg7E+HYz1aWCs3z8Y648PjPW7gbE+8dSYEVAECO+4FJRBPXS/R+ioZ/7cK2r/jtH1+OSf+28fExMfIs9y5SEJp+rLoFio8P4xWJ94UsznZ/yTr8TsLQOC9RVDsKnohl6vObZsMrVuKx2nLmsNRVZ1pdNZ1cdd6YtBsL46CC7yt3NAcLFkS4DgIrlWBsH6VwjBM7jkNf/O+ZQ0ZtU3AtVD4l99fTD8ANBFn+NgeNzx3PlgePJyX/CR6VuNvhh99fWZ8EL2s2z01RdHX/1hoq8xI/oaK0ZfYzr6GtPQ17h/9DUeH/oad0NfYzL6RiQEDwI+17cxPJLEh+H0AnC8W+R7aoUu89ATgwOd5zOgoYbLyOnhybtfHhcO44lw+txxptT8YeoEz4aFcvNrqYoFLYOS75eWT5GWGxM/Khqnr/hzovn8LgOYjdUCc0d1bL2jMqWh03rNoErNsWzNaSgdVavRmr4aYDYWA2ZjdcBc5KXnAOZiyZYAzEVyrQyYja8SmOd35GuCnouga4PP0R0Kb2RTNlW9XIH/nj8koD4lemMN1F8aiNJZmBGoxx3T3YF6dnewEGHPtyiKgBtVtAbuRexr2cBtLA7cxsMEbnNG4DZXDNzmdOA2pwG3ef/AbT4+4DbvBtzmROCW9iOQOsbsyJNofHH92Y16rNzyHfDXZPfk6HyfnJ2eHP90cHi4PyNyH4Ne4SZ5u3/cerv3qID7qaILhWaHzoDSOqhV8HGuWvEnAjQqGH7nKr7/42lj4vG0mL8x4k6ndxm4ba74fJp11HrHkU2T2bZp2A3V0TqmxkzbUmAFGqvBbXMx3DZXh9tFPnoO3C6WbAm4XSTXynDb/Bpx+05ufA3ccwB3XTUwU74Z2x4OYfeJsT6y/uIElM7CjIQ97ovuTNhzeYAFGHvqOig+xTbWp9gLmdSyodpcHKrNWaAa5gqj6+9JEOOCuFBTtWCTMcY5AErWA8W5Ifqn/cEfuUKgzkaakdnM8/rUcVy/C2AzNIRBE6NZ24ZI2y5GBpvCB39tW9LObuCwskVD3ml8cXt5GAfWiNltRWMIfG3PtUBcKH2W9PtBGN9WAXREkXcHlQ7hiudG/OptNSNmj1Y8ExeyOjt7YJeB71owQzGPsRdjEYJdMl/EBljUqPWm1JB2aGInPhnodmBuG7kdUoHmEozXT3rQBkzWXhr0IwKX8PMPfufLGqce87l0JkXPeT2RXd+4nM2lqmICbkObLByO16hlKMfXIuoEXiPHNMAyOMYftnZhrbVxfZEyof0+pExcodW+08khcVop4A8GjbUtiKIfpME+Aghw6XZXL0C6oSHVJK5AscbzdflHEPTaYhugLdTWGHzzgai6RZT+RxIFnuuQsGttGvUXRPyDtNui9oduGCQQYmDyAyiMRZQ63M9+QKl+ELko0RahFjSEGxzkjzL3hNB6rVbbJlkEI2kIE3ICB6IPyKQekKAADUoGyBD4vSCJWJDEzdIfzSx1BOCO9z2GL199OnA2iwcMIv5R4YOupHI0JZRD2ialvGHcCVhKy9jq8DykVgvW4EMsYe3ULjEGoLGMWnVbIgMfJZELhnmYeJ1aVbo7xTevQtyVGnbIT3IbKIin3KDa2aIasqvMmQ5Jm9krbjlBiukEFiMtOwwsGpPTvdeZ+85WXZrt3bZCUyias+Rw/BKdom2k3X9bLpOO65O9g58IhEq/jZtM7U4A/jck5fJOvp8gLsGsbOTxJb3WDqMIZ2sj26AYK+C5/od2lPT5nGbupXQz0OM+EU/l+YYIfwVt6KWdI9AQLI2IeNefu9RDXwcxXAwx73RCr1fMgoXFsGfoON0GSQtYLiBe0EYHW0pluojj/la1Ku7EFww8a6XnRhF077BOiIcrlU6Y74keiVuYE77mN0HYm5cKhH0ySdp+zwJJMw2lwlxdXVW6QdD1sOvMWwpnufMa4wZJfAZ0b18w/sE69JLZ5HteYbjvJ0MK8FzmY2fpRN8YPXYYuV2rAostH2qlUsFuuG/C8UE4hJ77QRISjxIIQ46LSfFfqLFKpTTm1aE3Lk1uJtxDpRk82Ri1yGpqdWiRaIX8dmaJrh+HAapp5IKYTeE1MEgWzAPmOaMepdSXSzdzsewOfteg2IcqgQdL9hj4cBhKXNr5ZoOQl567U2TJpZ2WbSfM9fKtlbHSfyshLEMuRDIz41rjVo2FoTj0m+0/g2MF+09K2S3RUqrdktDu8PxFN80X/Qt0xi7hR8UOeqWdXX6FnF3/dRmKbnNBZ+wA1qldce0+GCR/Xa04MCMHfgTGksRkFywBgiGeNYL6T2no3qkXK6kktvdppk4OP2FGNtoHbmFx7d+Yhsm9psZfsOKL13XxBFfx2WbMLoas4ntxaagSF2/I9rJXBcuAGziuA75mcru3KLRPk7ZSGiWC4ar4OdSwiIQz3xD1bz+Bc30XonNIu5vPt59Uq7lfCb/9Nj9HI1xOK3A+CYkv4h7fLfxm45uN/w9tk+kGFysBAA=="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514831.13/warc/CC-MAIN-20240718130417-20240718160417-00254.warc.gz", "offset": 3241425, "length": 6870, "data": "H4sIAAAAAAAAAO0963bbNtK/rXPyDoh2Y9pfrAt1syxZ2mPLauKsb7WdttueHh6IhCTEFMkSpC/tyfNs8xx+mu8tvhmApEiJcpw62e7uF7eWKVxmBoPB3AAi3++dDyp6ufqs8D08lS7vPNYhPhOe6wgWFR7QAApr1VqjVN0u6e1Lvd6p651m68eowTkzXd8qHR50yG7oO50w5FanXtset+l4uzTWa3qpMWatEq024WmnVm83qw2Tmc3+s8LAdQLmBKUj5kyCKaBpNnda82JFEPU8m5s04K5TmQaB1yUzMQmgqrdA6vfUN7kzdhdoqbV2aNtqtEqs3TCRAr3Uro1GJX1n1Go2GjW206r3IxCA2Qx9XyJ300AsWhvrdDwusVptDAOqNku0sV0tmc3WqDmqb7faOouBHJ6V9iwLiBMdold3yrV6o6y3WuVawmjqT1hQent+2CE4ok6lMuIjm7vBlP0SsvKMCwGjtdjYp47JymO/wuHbbdmben+zr+0eDYOp6xuCsXVu9er6TjOCfOa7gWu6toILc6vHFfTOdqlVOuATJoIOEVMKk3h8dPz64Nv9N6/3L5pvTl/VB9/+4/yo8ffLxmn9rHH0Kuq7b7vmVbbn/v728PC708H+SePk2736fqN+9P2P/zg9Gh60BoP9b4cxJyzgJB9zZiUELM/p7TSY2S9vZ/azwrPC68vLMyQbJK5KTv/+rKDk73IabhG9Td6EtpRFEklhk7w6vlwUmIDdBhUE2iXmlPqCBb0wGJfazwo/lAY+vbGZX7oEzooxPAwd07W4M+lA29C5YpaE5jATaeuQK8a8ErX5NQjZd9S/65A902RekPSD5tScMpScwEfGe64ISlBiXvWqW8TzWerLqqpnheGtx0FiOuSCBlukqpM31EEeVEm12pH/q5FegNwMXPeKwzjPpt7x3T4KTun0bG9wsN8zqcla9WrTSItTlzAFvHfsOlukXi29CZ0ScLEJHO00Yy52yTG9Le1NWK9eVT8L2F6fXQwvLg4Penpje9zcNmE5bVd36rSxA2t9e8zqY6veaDdHZpd4NJj2Kgv9Z6NTj5qlo9NXhyc9i9ksYNacNjnDVb0E44aVuZ2MW8/StkhUBBQpO9k7HvaigtWtgP5Wu7rd1lt6vZbf7GDvcm9/72L4FCKPKEz1MUgICv9H5feC+dfM7xC3Jm54YE5LZ+4N8/ddlJfrelpuY0mfi+3Iz9Gjje3GDi6nwu7zg9PB5T/OhgQXBDl7u390OCDFUqXyfX1QqRxcHpAfXl8eHxFYc5XK8KRYIPBTjLTSzc1N+aZedv1J5fJcLVRdx07xc9kKrGK/sCuBwxJ2RC+nr76zs6N6FLFRx6bOpFcc+8VkfWpygWoIiVGrX1jbnbGASiVWAhnm172iGY0RNT90VN96xVWLvUgqCRiHzlivqJRmquvZ8T555buhJ9tmG1+xuxswayLVPNLKxGJE6eUtkl5mywgtJkyfe6hIUmD2oz7rbOLTa9aFruR4DvobCTqHIt8duUGaHmrbEc7npdLhkLCACIDBHFBrCMl0Z7AO+YjbPPjfUimPqdoPpbd7IFOqoc20BLp2OOwNrQnTFsZ0zdmN5/pBio4bbsFqt9g1N1lJftki3OEBp3ZJmNRmPX2LzOgtn4WzuABkZm1tDUAHPLBZf0ADarsT4ARziM0nTg5LdiuqbWEN/9u1uXMFvord00RwZzMxZSzQiHQLNCkTphAamfps3NPKFdWmAiyZgcVRf8rQ4m/6dq3a2qk1WlVN8fIpcMfAkhK9YcKdsc8P3fWYUxJgtj4/aF/6cIbFhWfTu88I35uNGsbMGgMiG61qlvbt9meBPYGVwHzODG8K3s8XQRE/fBHYPneCLwE4oJMvwoxrjivTYaFg+eBBHxGoMFDjogOoHD0COiiN+uL16fnl4O0lORycnsTY+IyCm1kRPGCVMQWVAusUPqQaUtpUkan0/jt6TVVpkQjf7BW5Y9oh6N1UTcXy6cRwDMt3vfI7UezvVlTFE0BOqWPZ7LOC9FwvfBiYtgAMeULIGh+TjeeWa4YzYHUZ4oqhzfBR7N8NbCrECejtzd9IpQLuZ+gTzlDzfqQ96RFsNQ4d6QZvOO7MMLFy8zcsX7umAChgM9Fz2A3Z8316t7HZTapMNwTDUFUFY9cnG7IDlBC+m4/6kk4Q8Yb2P9pm2ZZuDDR++XKT/EYkLWtyoI/ojHRt8M2yOR9Mj8wHQNQIABzS/5Mk9eXLn2HEj4fdVRShp/RePak/PgsgYFSckYOX5fgL3P/GVcwklIQB2GTBwUzLKbE1ht4cCD5YO0Henh8JtIPzmS7Mp4LIpsyARhsWWE01IcheIH8ADhA4tOmxoE9EzYDJ8r+tKO9ky8GHktTz8UYaaI8U315+A55VJATRaBVBEMuiG+E6AEQRJkG8Z7Zg2ebgAXgs3USxaC71hU9UT7lr3n3nViz+jgcVUEEzqAxCBxRB9KcsAYCu+ugCU5x954KHPeYT4CwOxkMenTpHEM9CtOmHbAsKIUgG16ZDtLFfGvsaFnFxwEYh+OhjCmzAElCZZzblzhuBwWXcFYKOK1R7HfKT5BW6WgAIFO4+FUyCkuBlSKqVyxX5v4aM+xkrLeZhXw2iaqmmK8dMCAR4Ebg+02QbIM4eAZ5OsqgzjdSc3nDHcm/KAOIAxoyjfS/nKCooz6IuUIMLPwNh47fQt4G6Cn1HVaZi5lohOHv4fR2IZ5Ne3F/bIhYEVGZwCH5iR3LnvZQGHBJQC1jn8vDwHEmN+qAUxB+gXDHAiOF+DrDAlwTyYwnuo0i5MOWGmLo3hnBNcJQNhwUQb1wR0JGP1v+KxjwKBZ+BhjEEo745XaDuUUDzB46Rg+FjsCOCRajAWBW6FXZHrnVHXAfTPRgdSJGy2JiGdnAR0CAUPQ1DL5lKuf9dEC+EiMjUukVCOPQApiJ7io9YnYlWHAOWExc8DRZp7g1uSZFGZod2zwltuyv1sEmF1LKObC2ItAoch1SI9ehMQco3Cft3h9aGxmztJbdeaoMpty1tU4FWIGMvuiAV6PME3ObjICsgCqRUwHH3RNGHtpAAoooV5iq0FYyM/SVogKF/ZGGJMrFKPwMyqPmJ/4wA94LA56MwAEAcxofKX3Odic0CYXAxsoxozl5GbE5IA8oUFGX810Y+o1fq+X2i7UliFbCTzR/ih8IqkUpssWV6bvNN8ujuKXJjAHF/SbPNy6DUodcJ2LJuZLQjixWi5LxPCZtctzdTGmz8Erocgl1LeRUqdyMLFVNA1hjRhj+cHRlHpwONdCQfVtJq8WuD3Xq2AbpeIy8RblkawHIkUkCoNsJkrNZ9HKSVUBww1FpX6tsHwETMS0NKeVVES0mCwR1wJ/j1p0FNj3Ql5BhuIStPc+Z+HsYqljyRr4+anU9ha2bwn42l88laYGqkpiOGoiT3CH6WA/cIU5IDYHra05eLJ1f5dguxzx7aidOdxA1E6okVqsvmkerKUV6yd576kmta+fQfU2GG9hLHlNJdcTQAPR83Ex8TkjmGh2Uk4yEvjGFOyfM56RjTa2R9nTzcDCLnxzQT4WTyiGYjKq7AA59z6xHsSumDpNcqlj0wbWWfAd9MFsu4toX81TYfXsCxtZn/fb8WqaX3WYl/j6r9szhHAYVxC2A9ur+f4hA+AuhngzfjwlzyB4Gh6HhpNMDOyNae9mvpEHccO3q1WsXW0CbVVCbjIYxhfrGflM2gYP5Vw68GuoTMBwDTej9OfN//jilvixGb5mV5oaVClwFsTF1fGGPXDRBnNLdPYASmpow455JiB8CM0XLc0DOikCUamBR3cFFrxZjIDKmaAwtvQiEcwkhDQH9n0t8j4FoEXOXkWRAw+A6h/rUbCoz/r9mvpAP4VePCLk45IyqprgH7XwB1gQ+/VoRdIZH6yHWAJ9wHqQPyGXM00t+lKjAulhf2jOW3YhYGPAeRIwxYhEed/jkLXPB6mE9oiFuUM0xT3H8wYXYIvWYmgXgcRuI7/P53f+5Hl8tlGAMC2K1Q+A0sDLWQbinDLGYUpgbHzJ+hayWF4C8JyzA/mHBfA9igKwyACARoMdls5gV3WhrYXw8hBp/QEfV5UMZ9oK1ropfr5Vq5TmpVvYWnFqrbpLrdabY7zW3yzgPs1HEYGd565K8SKU4VSH06Z2cKAWJWBjGJ/XlDTQzowKgA4ncs6RbeF7INffdmuVkJSpebmsy2c9piMWgnj1q4x1fy+WQadEjTu0UIuyohi2THoipBMliRtpCLA5ebms6D+w9YLJebBs1Y6CfzBI1SSzoFw0gtb4SWbiIYi4Uo+S7XYyS4UZX8YmQaQAs/BgOrmQd3C3hkEyte3GmsUfPMTGSoyNb0E4uTXrQKjs2umV1LNVlT7ILve5I9a2tH9x9AJQ1eDy8uT78jccOIX+l+j/5ZICMjKXJdwiwjM/BkAcxgGsn8MVbCyzWPQRNDN3Bb0BXMeiyahZ9CtkH6i1r10ZOvhCJa/Qg/0zSxOSmpQ0snNxzXUhKcGU+6oRHwwIe5PoicCtBKHI8D4T6mkBqLQ5mH6WgWkBzRz6gaO6AAzZzGRGRwg19NHWvPtouJetXmtqSTVEvbv6FvdlVmJ52EMahtb0QV0k2/drkAs5apBbR8Nol1HTwanh0K/J3bMrU/olBit/KEjzUycn3QyT2tqskRqWqo1SJ1rIadGRN8Mdm5HPqFJKLYX3dGwuvmtTZd26aeYCt5kGqwsbk8jpnLHSE/FkcSd1w5lrjBU0fzRz0FwWxmLqW7lBRnv8rwBkXAmKBv7fqB3B5JZ80wdopbJ/mEuLURbalvWMyWBZhaSO1YRCGXjzHXw3lW7LwuwlEvBr3u018VRdrLDeSBO06wbEpXP8TjZqCKLQ23CaIq0iGatvlSA1ie5xtATaq/pG2pK5QmvdDPMZgD4wp90QMn5EX9YH507UWtxS0oweNrc799jWRZWI45B+Peqm5p2pa+FSVO5xyDiupmyvVfYvFSj4347GCavStDurEPEY4R5ffAKcJTELgdBG6Ej6dnelmS8QFFbGNOk5w6DN4eSlatQJMfJUeiG6Mg6keGzMl2G9+Ve8DRflo6WF5Lkv3smtob2AwDMKR6ASJJQqn5Q8LXK24rSTEk7RuLwiqrcBi9Txz0nHGYQkjAbM7JT8pSmTsQlpl7zWRWNtVpKShcXsgpXQLx54Vc864vdZ3rDCCgveppUojAnjIfJnZ51YL+JtQOesVLn2eMUZHIkyt5FVG0ZcL6AAcYDw9C1MH82BJkVaXUi6M7g/6qFCXFkzI9beQGgTvTyFTqwJ5WX1KhkkzcSs8qUJCUmTrWo81MA8QA1hlAlTOrtGE6fpi5PjOUqRRJmBj7rTL2JuDVT13Ah+ccMQDijhfGKnfKLQsDFIVPmb2fftaIRAsFRkOr9B/oIfliVA1sGHd6uIfrLTQffvtwhzFntqX6pCijnVrnN9HRtzvrv4Ru0BXM58D5X5kVGW9V3BWdmh41oR0du7SirxfDvfPB66jZSfd90kHXE5jSC0Ci5tCq2UpwJzxwqSWRImr1Xqs8yGZUwMlAdNm4grP+WOO52iGgY9T4voE+E4MeLOUWpCUnNTIDs0hWDzQMg+qEezPczgZwPfg1lCnTogWj7SksmClIEGn9vNK0XMsEGzIIY6SUnV69j4Q/4KhGWkgsu24/PVgbKdife41Wq9YtLGmYQtrlLTIbm51JpZUEMQp2SamyOF5KMzxGbjHP5lRpj1gzRQ4qbhbj1hSC17bk7jLoqWVfdEOTLVZ6o0k9iXZA5AZttyjPm6CJyfNSlQ9VLFdAL4J2kpPvuDN47GH1uivPQTrM7unF6Cwhnw1viymWHM4miaq0VNxajFVZsVqMFVyxjsceE/1YTPQjajiDGnrrVm+hhixKdazFAYI6GM4IrCOfz9YZNSEm6Cay9pFmlZVzUsIkFy0SPGc0wZnBCvVVJRRUMzVSdRgJCgwccTG7qATujmJCBliUdXOjNEtPzQyJma/Sa4YcAawLzK05EPxAvPPm/oMfCgqGNmf1xv2ULwaOLamQTNCb1+fH+k67nXDLDG6Na5gXKC2326VatdpYpzOv648DtCTGeBb08F2MF/U9d8xf1L6BAni+YtfwOQtu4XPkuldxH/AKHJ/1MkUjhQpG9WJQf7HXhpG9qFVZAB9vZMlONMKkx6d3QAUpYNk2oY2XFFvgRPf0nXp7HYcD05bUeOBTvylD4+987sxLMUPdgxXNRVJGQ0yu9SKGpoplvq53pAhyncUOF4PXR6fDH4fnS132XYAPqC2mJcKR+ZMsz5FPYnGdx9wL+kf6SIvqx1SFyrgXZ9SfcKek/ItOy7vtps09btdn1ApaGBK4VwycB2o1xu06HYFctKvtnfHOeFtvN+Bnu2rVLAZ+CfI44Gik9O2aXq/uNPUmZjpXZj9lJiq0pUOTsxWtFkYkq3lb1eq03BqeQEzByOuds+szN25/iZdA8Uy2QC1sKi083xfW1Dl7UMKrVKnqrKyWzaMZylKGZKykK9leyaPs8GL/IJ8urHmAKqzO0lRQccCaSuaEtsrqzPNMmu/ezPcHUokd3AVL83bBYZQ2uhtPOM5xDDBI9BmmvCMfV2ZAE0aMJqWJz+6yWRoNxOkXw5zCstH6l5gbkml1tT5AfFCEMu0jTj6oPJUD1CFv8eCaoBzzqB4sEle48IfhITU+4XgSNoNKJr6zQ7rzIAJ62pjAmCD+OJKKaVscZNp83X9YRZBKiIknUaQSphjvLzE6zxfMf3NNI/0F20P7W0SBJl3yeEgtgCR1JDIpUaMK3KVPrdBUKcB8fjBLfxIvhhaXFC/L3KoBSPkT4L7GY2huwwikFQGOvikTtDHJbkoOyXegdp82f45z/0HKlFoL8oDhKrFCe7iKEkfa0SfRcjjD10pwCzBHnMBAE28VcgGx2JNQfwMREQ1y0Opt0iTmbBVifI0pfOKwjxSMZbHB7dD7f1KQhY/2BZkHO03tFBhyHgrUSfmEyyThk8ge0OD+wwQQM7FKXlbKvcQ+X7cNFPrXh0enF6fwZ0gOhuRoj5wPjw5fHZ6eSPmX/syqwUT7J8HTpE9SSCDqMCGYsNknDyqmIh5XrQo2r9+gRzgANYX94Q+HF5fDk8vDvaPDi+MhKZHj04Ph+Un0ZXiLWw8OvjeFD4L4OIdi2bBEeyqF2OpKny9td+eeQ06apvt0W0d+ytqYnyF++PfV+GpcZfWnFKU0/oBSjvmT0sxbREYJOQikxlKPXRLpkXIkyB9Zz9GEx+j+kCJYgKG0QZ6r9VUV/DeogvgsypKKkJmtwnwzFTWE3Dc12C2bgU7A15rBeZl/yW6RRnv6zLajswg9bR6PpEFkQ7aCitmm85aeHZ0FUvkmcwSC41qsNKK+RBpMH27v4qbsAD4fahqAa2zYfATkQuuL0EOX4qEO8UsV805H+BYHF7L0oZ5CXRCQwqYK4j79gyjNj2/fSi98uiSx7Jo5WvpgT8KaxnajEQtatlLyQcpStjY79CN+7bPFJjmDZeSMAf/DWyJA9R5T31nqtTTQYzqhAtVfDnmCB6HEASIFtiF0pPzEak+dcDpI9j+SQ09kLrmFtOgmRi1nxzwTGS/X58ak2cRtBkKmKjlglMqe/OtSyO3HpZDbXzaF3P5oCrn9kRRy+1+eQm7/v0sht/9QCrm9MoX8d878Kzah1LfQObRpxgNksVlCTfCfnldODTXOEtsU08DzAWN5esgLieYnQVCZ53q7kZt5bv4JmefzvVfD00vAcFn+c5LS7S+clLaazXZN18fjVsNs661G26w39O12vcl0NqqNrC+RlG4/KSnd/mJJ6Vzl/Oik9Aq6npyUzqXqCyWl2/8BSek/oI6/5p//nPwzKBaApHQouSwvJiG+ZrL/rTLZzT8rkw0G/0/JZO98zWN/0eRVo57NXf2nJqv09o5MVu0/PlmVMlIfMUIe84UrD7A9adAn7ox4vuv5K4ebtpsX6244s7sQFH96Cj3HR4hS6J9gmf89suX/evv0pfPuzTwEUr8mefedr1n3r4rrv1FxfU34/7cl/FsPJ/z3vyb801mHP5jwbz8q4b9bSZ8gT7/bPZJhylSOzmT4RsLHLynCt5RTl8cETAQGnsHfwI/NwtpveEYdX3LB72UMP8rywD7pEx1fd5Ev1ICG9ION4kkI/oPvyjgH3yEHbnrk/oPNru8/kOfF5MKfCM4YInURvYSUSaoUopek8tDuPhLtiIpPRpmU4caCvFVHvvOdXMmVen9VcRsvgUq9poKPq99QWfadQM3Cmsf7ZEWvtm5HWZAeXXhVhbiOCEczHuC7oIq+7CzJl1mW3seRGVCp8+P3cQJ8icSPXn5Zao8vNy02ly88uSHeTCZZq1f0vK4OiNViVyzLR4Q528XWWKYl73/Eju9uJRLj5HuyhuTTmtp7ytTO3+VVb/NzhxwcfkcWb2nAN+uT9G5ycUMhMUtRmeHLN+3x/jCVL15qgLcBGiL01Lvoq683GLsydpcpavkEMLaL/WP0w2GCiQ0Ok/LSwEFQBjhBugLrDRvhTaSIGRBHkhk1UHc/GyiYxYimx93ln2xQLd/63M+5D2OJ2LVV1HqzEa6XiEOpa7gnrjuxETUJ5L880DNGEPNf9b9BU05Ch83fo1F3PLySHdK411IMsDlzEFl8HUd29IhQ8Mmo7LAgGWq5XEY0wrVDqQXBuABmdfslJaBdLE6jm0GgaTGmU1OEan3AJqlJxEQmgSOBJIWsRFYiqUOJVPc7pDYa5K0exf7iNR9yzqKLDMARyr+XZCHNXPT04qLViWtwX07tCxRnzAkPmO+GMJSg2H9WIGTX5rkXdRT7e6YZMm4n+e6l1n8pYtoTb94Ype9TkVL9TF7ejnjj3UDQLCD/YTGuUpAi7hYVd9PzJxbFF9UDIGPgXoiy6c6K/YEsIRf3v1/7Cm1C6CMR4O1+ZW56+O9a4HOlbMGMHDoChCUMCIQ2U9fm0SUxMgT8I1hGYTk07btHITm6c51FHLivILm/MA2rsUbCn7Pi89d1/gRX8J/ZwH8XIyUVr1RRqpMkb1Fb5y8DKeDy8uW19EIYUYBPQ2PxKpt01+R+DrmMVP+H3R28+N3AHWrwAdYqlUSv+M+fJ2aeSDrxlkhFMd7agk/4Dxb8HwFw+UOTZgAA"} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514659.22/warc/CC-MAIN-20240715040934-20240715070934-00517.warc.gz", "offset": 3365888, "length": 10778, "data": "H4sIAAAAAAAAAO19+3LbxtLn32KV32HC89mQ1iYJgHdK1ClZomM5sqTokpyTVIo1AIYkYhCgcZGspPwA50m+uL6qrX2G1dPsW2z3DACCJEhR1zAxfJFIoGe6Z6anp3/dg8GPOye7JaUoP8v9CJ8KZ1cj1iIu80aO7bHw4h714aIqq5WCXC8o1TO52pJrrWr9p5DghOmOaxT291pkK3DtVhCYRsuoyLRWlllBqfeahUqlWik0dK1e0OqGVqFauaxWlO1nuV3H9pntFw6Y3fcHLdJQmrX6+LIQiI5GlqlT33Ts0sD3R5tk6PV9uNWeEvVH6uqm3XOmZKn0elTTWKVglCsoS0MvNBSmFSp6XdXVSrPM9PJ2WAVw1gPX5cydZCVKtSwbrNYrsEaTFiqyIheaZUUpaLVmr2lUaR3+RZXsHxd2DAOE81pEkZtFtVwpKrVaUY07mrp95hfOT/ZbBFvUKpU0U7NMxx+wjwErDk3Pg9YarOdSW2fFnlsy4dun4mgw+qd1YbVp4A8ct+sx9sI02uW44mPX8R3dsUS1MLRKdINeWQ41Cntmn3l+i3gDqrQOfvru+OD7avn45M3h9++Of/hx/7vjw39XDt7tf/f29W4nGuDXlqN/mCz5773D1z90znfPyyfln87qO4f13Tf7u2c/7u/X3p5/t/9j1BEGdKTZM5kRCzA7pJ8G/tB6+WloPcs9y709OztGsUHhZHL03bOcUL/3jv2KKFXyLrC4KpJQCWvk2/dn0/ris09+CSvdJPqAuh7z24HfKzSe5f5V2HXppcXcwhl0rNeDDx1bdwzT7reANrA/MIPXZjMdZWuRD4yNCtQyL0DHfqDuVYvs6Dob+XE5IKf6gKHi+C52/Mjx/AJc0T+05Vdk5LLEl3m3nuU6n0YmKEyLnFL/FZEV8o7a2AcykeUW/ydaegpqs+s4H0xo5/Fg9P7qNepN4eh4Z3fvdVunOoNZV+0mtWmTMFF5+41rviJqvfAusAvQi1VQzpYa9eImeU8/FXb6rF2WxZ8pbm+PTzunp/t7bbncbOj1iq43gKisUkOvNutMM5o6fK5q8iYZUX/QLk2VH2pHI6oXDo6+3T9sG8xiPjPGsp0NAmx3AdpdUJr1uN3KpGzTQoWVomSHO+877fDCfCqQH6a8Ktdqar2RTra3c7bzeue0cx8hDygM9XvQEFT+G/X3lLkXzG0RR/UuTV8fFI6dS+a+dlBfLspJvY00fay2mptmRhU0Cc9yua1v9o52z/593CE4Icjx+euD/V2SL5RKP5Z3S6W9sz3yr7dn7w8IzLlSqXOYzxH4kw+N0uXlZfGyXHTcfunsRExURcFC0eei4Rv57dwWrxymsO21U8oqzWZTlMgjUcuidr+d77n5eH5KfIJKWBOjxnZubWvIfMqNWAF02Lxo5/WwjWj4oaD41s7Pm+x5UoqrsemQtfPCZiaKHr9/Tb51nWDEaSeJP7CrS1jVvAR5aJSJwYgwy69IcprNMjSYp7vmCA1JoprXYZkXrO/SC7YJRcn7cdVveNUpErmO5vhJeahlhTy/KRT2O4T5xIM6mA1mDWvSnSHMQ1MzLdP/f4VCWqdK/yqc74BOCUKLSXHt0n6n3TH6TJpq04XJLkeO6yfkuDQNmO0GuzB1VuBfXhHTNn2TWgVPpxZrK6/IkH4yh8EwugA6s7a2BlX7pm+x7V3qU8vpQ08wm1hm307pkq2SoM2t4d8ty7Q/gKtitSXPv7KYN2DMlwj3CiSuE7rnSWTgsl5bKpYETQm6ZAgrjvhVBIp/KnUwBU21UpMl0Zf3qbcHXVKgl8xzhuzha3dGzC54sGw9fNUud+G6humNLHr1gPWPhlqlOzR6wMjCVXVS9nrjQeruw0xgrsm6owF4P4/CIvrwKHW7pu0/RsU+7T9KZ1yYODNtFngsvXqwRwRudNHiogMoHD0CNijJ+vTt0cnZ7vkZ2d89Ooy4mUMKbmbJM31W6lEwKTBP4Qc3Q8KaCjGF3f+VXlBxNU88V2/nTVu3ArC7iTslw6X9rt01XGdU/NXLb2+VxI17VDmgtmGxB61y5IyCxZVJU5VhnxCyZvbI+jeGowdD6OoiwIqOxfCj9/pq16Kedwh2e+N3UiqB+xm4xGRoeW+gJ22CVL3A5m7wuu0Muzre3Pgdr69dUKjIZ0OvbbNLsuO69Gp9YzO+pTsBLAyyuNBzXLLOC8AVYm6lsz6jfWS8Lv0vaaNocTcGiF++3CC/Ey7LGm/oEoVRrnVzo6iPG9Mm4wYQ0QKoDuX/mYv68uUv0OLl694UEqGn9Fl8Er9c5gNeFD3DG8+v43/o/TeO6ExCSeDDmuyZsEzzIbEkht4cKD6sdh45PznwcB0cj3RuPBSEk7IuEK0bsGqKAcHuBfF3wQEChzbZFvSJqO4zfv2fc663Jq+DD8WlN3vryUrbJH9+9gY8q1AJwtYKgQDKohvh2FCJEIxX8ZlZHpskBw9gxJIkoovGWp+7pXlKnfPOr07JMH81/RKYoCHc9AMbDEH4q8grAFt14wQTPfurAx52z+xDz2JjRthHR/YB4FlAm27AXsFFAMng2rSI1HMLPVfCS6a3x7QAfPQehW7AK2Ayjy1q2u88BJdRUQAdH9DstcjPvK/Q1YKKwOC+ph7jVfHqOSSVisUS/ydhx/2CNw02wrISoGpupkvvmedhhae+4zKJ04BwlgZ8WvGkniASY3pp2oZzWYQq9qDN2NrPfIzCC8VhWATu4MSfqGH998C1QLoS/ZWKQMXQMQJw9vD7CxCe9dtReekVMQBQ6f4++Ikt3jufuTZgk0Ba4DrWh8VjxC3qQi2IfoBxRYAR1fsQ1UK/xDUvK/A2qpQDQ971Bs5l13N0cJS7NvMBb3wgYCOXtv9CxjQJPXMIFqbrMerqgynplqo0veGIHLough3Pn64VOlZAt9yW5hhXxLEx3IPogKuUwXo0sPxTn/qB15YQevFQyvUfHhkFgIh0aTNPiAkloFOxe/JLzM7YKvaAy6EDngYLLfe6aXCVxs4OrLYdWNYmt8M69biVtTm1R/iqYGKTcpEdHYqa0peE11f7xrrELOmlabyUdgemZUgbompRZeRF57gB/SaubmO5mkUlokpugKPisaEPLI9XEN6Ys1wFlqhjYv0luABD+XCFJWKJFfYZmMGdn81fsMId33dNLfChIhPah8Zfcuy+xXyva3qa0Q3H7GXYzbFoIJmoRSz+a5rL6Afx+XNs7Um8KmAhy1zUH4IrZ8q5RSvTN5a5QZYunhA3qiAqz2W2zCIYdSh1CGvZZrhohytWgJrzOaFsfN5eDqi//jFwTAC7hvAqROyGXxSdArrGiNT51/FB9+BoVyIt3g9zZTXMiy77NLK6YOsl8hLrLfIFsBiqFAgqaRiMlTaXq2luLTYs1NImt7cLqgk7L1lTwqsiUkITuqYN7oR5cbtaky2dW3NUb25Sn8ad+zAdK7rknv261OjcplsnGv9gXToerKlODc102KGoyW2CP4u+c4AhyV3o9KSnzydPqvHdzEU+e2DFTneMGwi3E3NMl2WGpivFePHSaeaLz2nh099kwrrSS2xTwnZFaABKLjcSNynJmMNiHZnwkKfaMJbkm7HoiOkl8uIFWUwGyHkZMi/o95cg06j3ATzwcW8t0V0JexCXmtdlC4at6DLoN51FOi69wv6VNhZP4Gi1Gf/+vBaapc+TGv8ZTfuDOEc+hXZ70PXo/t7GIVyi0gerb2h6+ow/CB2KjpdEfSyM3dqWfivsY8KxpciyjNRAkyDlwXiAMczNb8fXhnBh/FXCr110CZkLFQzK21Hg+/oPDHkbjFg0LcoLlILdRMXdgeN63Z7j+MgzHNt7dASGprpRzCXRHVBnxNbEhF43hCxhw7i6g4uq5iMhJ0SVbJh4fQpwCJGGB+Xt/vYOAdfCN0VMnvk+g+8A9S+cwEP8f8F+Iy3gL4hzWzjkjIigugTd/xyk8134b4TcBRNujxwb+sR0QetAfMZsiWxvUQGM88WplDH/lp+sAz77oSMMXLwRtbdPmO+A18NcQgNMUQ4xTHH9RYfRIfSC6QTwOLTEtc3rP9yxH10sFqENWMFWicJ/30CohXJzHWZRR2FosMfcIbpWXAn+EXcZxgfj3pegbrAVXagRBJAisdlw5F9Jycr+ax8weJ9q1DX9IuaBXl0QpVguqsUyUWWlhpsW5DqR661qo1Wtk19HwJ3aNiOdTyPyX5wpDhVofTJmp3seqFkR1CTy57tiYMAGhhcAv+OVzdzn3CSh61zOkhXg6iypziwrhRYvg3UaUQNzfAXX7A/8FqmOPmENWyIgi2JHqsqrZDAjLY9PDpxuYjj3rr/gZT7dJCBjgRuPExAlpnSijm5iemNtSRKPsUiJ4u98PoaKG97iX7oTBEDhRtXAbDb9qyk+nMSIJneSa0g+MRITUkze2Y5XnOSkFfVY7IJZaoJkTXQXfN/h3YO/+4Hnmzb5tnN08m2HRKRhjyVLLv1nSpAJXeEzE8YZuwP3FsAYJpmMP0ZmePbOMmyi2ruYGHQ8ZizLZupPbpIg+UXM+/CTK9QinP9Y/wRpvOok9A7XOp5yXEvo8ER7koRd3/RdGO290K0Au2TifiDMZHrcZplwbYQBaeaTFOWfMDaWT6E2fRAJMcEbPGtqGzuWlY8NrDReTVrxbb76rysbmyK2kwzDdKllrYc3uKN+4ZgeLGwTd4GtOexH1g4+dkdW4OH/8WomMiSCJRYr9s2eRDTHBavclmSJt0jchrtSaJBFsyfaBF90dsKbfsqFyG+/sDVvtJlGrTuWRUcem9sHCYL1jdl2DB3T9viP6ZZEBee2JSK4b2vu6it4zGL6TMBLaPHkVw5wUAW6ffSuHdfnCZJk3AzRU0QdRxQi6m6YVF83mMUvYHAhkbMIQZeLqGtxpBULv/ACrR1V/cKlvwmJpJfr2AdOL+aywZ39APebgTE2JEwUhLdIi0jSxksJ6hqN3C5IkyjPZZspClfjUujpdJkN7Qpcrw1uyPPy3njv2nO1ZhpwpazKY8d9jUz2YDHqOGj2K/mVJL1SXoWR03GHwQ15I+H7z/TwTIn1aO9gsnfnYrqeCxCnGwb4wCvCbRCYDwI/wsXtM+1JkfEDatj6WCY+cojeFkWr5rBJh8mh5kYsiPjDMXOcbzO3eBI4TKgl0fJaHO1nF9RaRzJEYCj1VI0kxlLjD3G/fjAtoShdLvv6tK7yW9iM9i0bPe44jCHE1WyMxY+vJUJ3oCxD54LxsGyi0AwqnJ3HCVMCAPSUT3nH5abOsXcB0X5oS1yJYDllLgzs7KQF802o5bfzZ645sRblCd+6knYjhFs6TA/wgHH3IMAO5kYLwaSl5GZRu+rS34SdpLhVpi1pju87Q4kMuAlsS+UZC8rFxFz6pP0ETRmKfT3SUO+CGsA8g1r5yApjmAQQQ8dlXbFSejFOjBxXDr4JuPUDB/jhRkdEQKY9CiKLOzANAxGK4CdWvZ9/kQhnCxe6Fam0vaAE75eu3EXCqNDiEs5oirzz/eICPZNZhiiTkIy21NbvXkupt158DBx/02OuCT3/GzPCtVtc3vRaqhKS0JaCRWrh19POzsnu25DscPNzXEBR4jq5E4BCjWuTJ2+CNzECn5oL6YVUn6XSwm5G+xs3ROHEJRz1ZdfO+f4A7aHBd7voMjEowRJeQVJzEi3rYhjJaIOFYXA77r0h5rOhujb874qVTAonjLQjuGCoIGYkbaddTeo1j7BhByFISizT8xNJ+Af81NAKebOe288L74YG9pe2UpPr5c3cjInJJV3ePLM43TE3WzGOEbUXhDGLIFOyyyP2BhtZJhX2I7JNoYeK+WLMTvH6pVc8wwymatYbXZcEyVyHdExAwjwIT9Nu5vmuE1xn0jxV4UfliyUwjmCiuAbYzhA+tvH2C4fvhrSZ1Vby4Y5Cc9j5lE92y/6wHxtMQ8DXfGTQ8nI+MnP5Mu5+jK1kPraSaOe6WmB1ldonpYaWMs/NsvQ6sCwGuE7sEGcE5pNrDl8wqgM02Ix17gay0tyRKWC0i+YJbjjq4/jgDfFVRBYEmWis2JUEF7q80fnJ2eVhnhRDM9BNk+5u1OMistblMkvbVVIgSrNehl9HsG5pLou+HvMdJqaL+P/6ywW1+yagdHBp0SkC7O2Tg0BP8sYobtgDTg+XoW2yHnUJhhF26cBkLuCr/4SVbWyVzFjGWUsRSSrcPvChSYlMIey0Uj+Vm41GPCa6/6l7AeMPV4uNRkGV5coLOhxtuj0f161ub+i38cmP5+Udp2c+V9/ABfj8gV3Az6H/CX5G9EUaYNirLVgnrvJAWjsSTZpukvgVK7/mkkgTxqh2eoZzP2R6guviolhA80Pq9k27INbwVm30aTO5pGJOfGLeohUnvvOBwQJdr/UUVq/K1WqtZpThZ6VSqVV6vXK9ZlTKKrgBBmAB38SFQKmrigzdptQwnDg3xMjDPYHFnYaUfK8wCdEQpSWExZ60Ndznl6gktXhKcmW8hPwjGvr8MadAS6cLSzdOv0piOzvYubnGSpQWi4NlhqM0KRvKMV+yOI+RJtv+6eu9dMnwziK58P6kVDnhcK+JoElgiejJOJ4juc7lOBKfCKBgvmmif6dcM74abk4GNKSeYxsFDEvraD5+tv/v/yFOQKKJ/0uo7eHynxAictP5IoWAmM+DLW0bEBoJ7LgGsBRbJW2bpPoE6E1ObpbAp5gU+YXHEMYq0v2M4YyFehr7grQ4HtvJQHjYPfynGLqtMB7uu5Eofiw5BvlDp57HfGON1PqFvsuupgYROvpjVx+AoNL2GcbCeCJB8Ia5jPN5gj5U6QdaM5KceKR/skVXI0B892sSLJq44kTIUWhji0y3MblMX3+ZJ5CI/3n3kkhEiDG8MdPP8/R85kk9iWzPLH90+xXZCeOT6dJfgfW/n+i2ff2Fd6fQAr6ZcF6Poh7Mk8Qe8R1895Flf4iPkGC6L6Un6woZzePtAeq6F+c3gH2on8JVaRJ9OI8tPq8U3LPNB6KO2RmKec/r/6amR24s6wBTWI+sRDUkLj5PeB4PvJfou9S//tIH5sybpzBz1Z9zj7RfwdVp+/Tfh0fHZ/vfn3dOEyZyjvRhbsS/n75xkQiACR0wgsVu3YpIiqghVVWFhuwqKL8YtnjR6UytOXPaNQJah0P5+2mVeQFWmj+G9jGY2y5urWM5woRMLnIluDubdCYSDlFKlCfzJFbFk7jnAk5+nlw4fwFg9lDLmBjuovhVEL9QtrTraPDFp03CzXAxbOUN5nBCs1rkTnZ0Xh2pTlpmSTNLmmZJo51AMxaWhxVz40Q2zmmes+6yT2wI1gofKpe2O+Mvk+npcEcFs6xwJ0hbUsdbYRJVTGL5nADzgzHlyAp3YolAn66BKjoGK2jU5Uz9wWJ6BxPiu/BzEakPfnrXMjUQF6hPgxE6eYsKRI+0jAsd4DM0psevLirpieMZEtzEhajM9l6YY8FnnzkkGMyoDrtgtpTcVhV3jVJTa7VoyCfvio5Qpm9ONv2EXQRsmiSlsYwcM+j/4BPxwFy/p649U2qmoe9pn3qmnSqdZ/oB5wEqBTY9sLn+RMZU7C/bi5NP8ZYzkjqzHMOY3zv1r713FitP46vsnlzS7sUOZcpWl8lg2yxBapRrMucyWcXEvXh3YCIs+2TpH1muL5P9AbLHTP5A9TfkfpBiUepH3H/KzA9wfJDED51J+0Tbw25I+9xA9vBpH2jx8lkfUUdbjMwctHIgUdvGTX7ECMDNtwIfwz17JgtWOC2jOc6HOAnTZ7bL2hOXNMHq4LlaD1v3XJWNAH7wFuIXvIKtjMvcpQiParWVpgp3RvFlTKDAxVo5wmTxHdwFAAV2gNyiWOXz3fLznaaOG7pdn3NkMfEo0Np4v9H8GJgj5sFdZnsMHFVq+4AFYjp81KENpsH0njpdBXr1uNkqVm1qDVrp1WuVXpVWe1pTo2WDUaZU6opSrz1CtorPlTsnq5KlHzZXlW5dl01VzZPrvpmqdKkeJ1HF2zAvT7UqCZL59jSCh98GJgbUebSFEtwyFj7VkAy7wNcsXfLE6RJmKPeSvGOYWH2KuswTnKuON2Bj2asgOjfl0P7r/wjDT5JmP35SZgXyPbW5+R7cfX2/oBdUIJDIbRQB2cZBohpqAU4xDDyFKyyPC9M5+OwJ8lTgKvwpiapGlqh69PCqqlZA4/YPz06O9s53z/aPDsnrfVi+vz/vJAKtX1ekVsF27JTlRKT2ren5jumKY4sMpz9kXpHg6an0hambrr6JKy7MWnzykxWJmMERokrM47CQ6Xl8rhnjQhxIr3b094Y8WpqnE6bRHtS/eOpszl0WwqhZc1fDV4RjrdTkkZrIEqENnLy/fu+FZINkeafMMGaGMUuLZWmx9MyGXFmU2OAzYFFmg6vS3zyzkYzN3C2xkaxhZfIa4CHIS+Q1kOwR8xpY/eK8BqdYkNcI7z9hXgM5fl15DWzx7fMafGTm5DX4goeL37vrL24AjjIbrlJqgosXJRXeiTRAKOZURmIpSpGIKDeaaYmIqjqTiMDcwi5ze8skEY5Oz/49k0LowBRi05RnJ/uH4JSdzVC/czw2GjxxdgK143GzEzXFKFO52lN6qlFvqIZBNYOVZU3rMaUqU/bw2Qmh8XfNTkyUftDsxBwbuWR2Yq5c98xOzJHqUbITog2rnp1IsYqRe74zdABvHHnXXwBBvKUa/Rh8BJShGSaF6++c6/+xiuS9qQ84wakzGoDvA58O6SAYAmG/z2/8RPE0aLzxnoJNGZg3bcXLkhhp4ZVaHTEzt7IEDbAIdJy51Ah0nmTYXDZQU6tjKEVYYBJZ6LtXtyDuM65uhTMsaiLBgivgKuVTquqflZcAz+HPyEuo8l80L/H2+ovm4haZv0BWAtzbbYy3df66sTI0ObW3iVhZZ/dk/+z8pENOd/YPzzpknZt2b4MUyHiJi+BKuNCRF7ZBvcFmRPHC9T4GzuaObbgRJVl/e3R+svP9+X5MfHb0enYBWyqKn+oRhFH8P2Mdvkew/6+9Gt0vE6FOJyKiVQPzDgCtUirn1jTKO3Ab9+dlCSJLtfI5gsxMPZaZymLqf6+YelmtNRfE1FEDv/KQ+kRA4U4h9YkaViakrjZlubFETJ3TPWJQnde/OKouSBaE1SOCJ4yrc5ZfV2CdN/n2kXUxOPMeGaBEH7iYrbacvskXsZkDPv7CDw3gxvxk+6LYO28j/IZWTj80cKsiIlY/cqwr3RmZIqAPNA3xEMF0pP5pYuV8vB83WK5rak01DBk382usVlFZjzWrTbVW79WqDUV++GB5qMR3jZZPFn/QcPk827dkvHy+ZPcMmM+T61Ei5mErVj1kvsjaRZgB5gru2ZFgOQkAp+Op4jAfoBcvEJvhYeKgwSPX6THPS54nnkXEn2Bb/2PHTiNLfv2FNLLd3beJot64u3tFgX4FNbH6w0KgP4h8OmE2NmKkfnh0/kNn55ycdU7Pdt53Ds9IgZxe2ei64pHlZB3E1gN7445Rx3SrGm0efnhb9tRbiH/2ikbxl7Q7U/Nwct/viu7SzRQ8i1f97eNVSqNSWRSwQkX7ygNWkx79nSJWk1WsTMgKplBzudMtmo97ukXzxtMtmjecbtF88tMtml9XsApbfKfTLZrzQlX8SYjr/42Y7931Fy/wCrvc//nrh6l4y3jwqAOtG28NHbdxKkh1mwLhuRZqOfVcC/krONei+cjnWshUqfR0QzEqjCpNo95QqFGWK3pFrhlN7RF2jopZctdY2ETphz7XIs2uLn+uRbpc9z/XIk2qxzrXorn6YbB5lnTyqVN82Cx84FQ8gMrfqRc+dOoRbwwDshhYdrTFKh9tIc/d25cdbZF+tIVazoKfDxn8XKndWdkT3Cv7BPeyR0rMOhlhVPihl/a/z6kScuqpEmjoslMlMpuU2aSbbFIWyf+bRfLl6sx5DdlpDtOnOTTvF8ifqGFl4vi18jKHVAPVQ0TxgZoO2xJ9Xt5RFPjxvP7ag18qflRV04Afz8uvTfgKDOET3lRq4m70lumu78ByZVmCllPIggJfGSu4ju81wlu/gkKBKyBucP6VMf+IBZLwd9jOVFMJWRSfq29iU/28/Ea8xHzypTPRi8yfqwr8w0Yp/J861aAxNxGDGFOUa4u4zb4yfS6n+jQnJvB+glVzEasJR+smbmplmpuw/IFt4luGk71ZWcR0stRS7axOc+aB60Qrq4sYcuKlWihP89FjzJ9gtrB1ka94I6/yTJsCLZVdfWHbRKGlerExzTFyQxLMGouYJd2W28+CoePrVmLQagt5JV9ADrzwvdFw9QO7unRcYFYLYFS7gFrcq1gIYBBJUcNX3MMN58MiLeLO5LJaFHuet9fWESig80jGZmZQo1dpJfipi/hF9EtznDE6Oh5rOPKXVdmQfKmubKaY0gE/7WxZExcXuInf8/oe1laPFxSduv54wZIjkZSYABYqAMp2GlElJgr9O8e2rlLo1JjOdkJSQaWls7Qd8MvsYLhYLv769V4v0Y/iXmV8JdRRntZKdGUoT7nBaqymaYZuMNo0KKtTvdyr1xWVVeV6vTyz/kZZr7Fc4+yX6FuJpOX4u/RX+gkT/eAPhHn+V/7A9Ip95u/4vmtqsHSuS9yvkDY2NknqHgAsvGALgLg9uwNAZIAeYQcAMFx+A8DfIP8P7b19+h9HZU72f4cfeQog1o8AKzinjnjs8u/xhguecX+u1uN8ux8/e8IT8aK9C953cZ8Kol0C9dRdAtV5uwTeONBk+cK8wFjbnU6kumv6P5yoN6X/QaMeJ/s/FkHAuDCsk7/55ZQ4MyLGk2y2ASkPqWXaHzDoM8ZkfzoufPJtZrXltpnVnhagIscMoWYINUOoGULNEGqGUDOEulIIlTWbCq0CHK3im8YadaUqq2VN1VWjqStl1nh4hIoOwd0hKi+9eJt67WlBKnL8qlAqNvhOu9Rry+PU0VeGU0f3xamLKghxallOPRy58Zfdzb4knEXNWzU8y+fQagLa+UjyieWoq8scn49kT4xo66o85fpmiDZDtBmizRBthmgzRJsh2j8X0Sq6rIEPqjaNWo/qGiAcVVZYo1rrNcpKuVF7BEQLDsE9EC2WXoRoxf2nRLTA8etCtNDgOyBaHJh5RwRK1//pg3vCCEAmD+AtvWC26S4JZN/jwdWFPWdo2vhAAdl92zk8XyU4exAi0UaTNxLTmdDMEKKKpk4fEXibEiFgrVVTE6vNeYC143k0gS5vk1HlPTyDQKfGYbrQ3vn350e7M6XEwyMmtW+Jc6fJjzsnJ+e7R4cpDKhrsZk2nOzvHnXOT2bIj+H+NO3rzunpfudkVphvwxcsUde4N+6G+bFyuBtn+ori7rl494lfW1etK8u8tg7InhZ3I8cMd2e4O8PdGe7OcHeGuzPcvVK4u9Jr9ORyWaszpjTrVblW1puMNqpVpUJprfYIuBsdgrvjbl560Wtvxf0nxN3I8avC3djgO7z1FgdmLu5m3uj6iyue0Q3IiXNFoUnL4269IB4AB10gBzu7O4erBruhfRFEs3Weiw3gR9jOFNC9JL2A3KqcDrlr8yC32B7tRXnnC3MyMfyeJngsgN+8p9PgtxgO3oC6c+vM8TT5+6ODg53Zt/PuOTb1YbWYAclH+6fvd0725sQFUl8BvHd+fHSY8gJgqn8UB7HcC1Kj6q8apOaTeDUh9Xwo+8Ry1JuVZSA1kD0xpAaOGaTOIHUGqTNInUHqDFJnkHqlIDWrNTVZVuvVplqt1SpGrUbrerOsaNVypVntJep5MEgNDsE9IDWWXgSpxf2nhNTA8euC1NDgO0BqHJg5kPr6P36AZ2fhSUSW5LDggh+m9fd42Z1ISfMG4uvq0GXjGWnRzAjWzr7x7g7lBMKuNFKfFq7P3YV96gSuzrloJniJiCInIPY7sLfyt1SjlkEfYLP1DPn5/sFB5/EALSjeygFanEIrCmjnAsknfui5Wl7mMKxq+YkPw6qWMzSbodkMzWZoNkOzGZrN0OxKoVmVVqpaRZVrtKo2qyrVGz3KerKmN3uGQWX68GgW/IF7HIYFhRcdhsVvP+VhWNXyV4Vkob13OAwLRmX+luzotHaEr7gr9q+PXw/iR4LDpkXQc2LT761oF59qNTcTvPqnWlXLq4Y0UcVXE2jOBXhP/ixyY7lTrRpPfqpVI4OaGdTMoGYGNTOomUHNDGquFtRUe41Gs1duaFSh9VpD1mpUN3qKqihGtaY9wrnL6BDc61Srxg2nWjWe/FSrxlcFN7HBdzrVqpEBzgcBnGUl9XiqyldwPFVj1YApnwyriUznQ8KnlaPSWOZ9QED1tMAU9xdkuDTDpRkuzXBphkszXJrh0lXCpfVeU2P1Bi2Xmdwz1BqVy9W6WtEAphrNBp1df++NS8EfuDssxcILUKm4/YSgFBh+VZgU2nt7SIqjMgeRirdbAxjFJB3xKIHFwCcGtT1+AHMCrOKjl0uC1d23Ryd7O2fkXZEcFFcJqr4TuBBazCEhthn36iJkxHbjRWpHR1Cl4lTshSlM+9CVhuBXbqbuCpbngd8D042PfuYvIb7jqVd86KAOvov4oPjE0BdUddWQL0651QS+cwHnk+Pv6nIp2eqTp2SrGfTNoG8GfTPom0HfDPpm0HeloK/eUNVms1zTevCjaijNnqEpVaPOGnqF1g354aEvOgT3SslWb0jJVp88JVv9quAvNvhOKdnqDQAYSF1GLsxlz4Va4XzsBB7l7Zo6hmlZshCIVhtf7UuCqquGRbn+ryYYnY8Cn1gOVa40l4GjSPfEeBRZZoA0A6QZIM0AaQZIM0CaAdKVAqRavalVNUy/ltUywNNetdo0aLOhqRW5zGr1RwCk6BHcA5Hy4osgaUjwlJgUWX5doBRbfAdUysdmDizl58mGeVhGDhFC0YCcMagXm/bXR6m8gYncKH4Omwmf4oZOgdbbljI9zW4369CSgqrKBVktV+sFZRbiVtIgbkOZB3F/cEwdn231eZKXw9yGO4lt95inWwIAx3ne164TXDL3AbDtTNp256fOwUHndKbAWxgA895IGDV15aAwn3UrioXnY9CtkmY5+oePgeOz7VzUyZJNLzTqwkANcCy2dFBi5sL9GyTe+qZQyK1FGBLsned3EV+s44+N3NrvOWBp9gj/XkRtL8I0DhjZJmW4DfdBKmox11/PHwbD6y+ug3s9kBDWJGdErr9Y7OL6C/kmv8HbM66nB1bWWxdXJ1YXuPB5HtstoizFVqPerVnG13AxhUufc4UC9FDU/7yrI+3lvQ3L4BZWLhYxCT9KhPKebEvTCjiGp/xAMFV+YWsuoO++DfagrL6w0P8AL6VNJTJkQAuDOnI8X4L1H9DK0IT1K5JvcpRwGKMlUIqXQD59C32XXeECKIkFkF9Mpx+57GKaHK+ZDqzo2znetVumPQqSyiRFqwD4VswFvw6opLA7xGc+bDhRYer+xj9sl8q5aJou0UtYTVud31nbKW2xQbREM2wuaeg9SHK8WPE2eYE4XwBtHEz120pWvp1kaGwTkuHXNMngq21e/+EKlY5EK+Fw429uY7dK4SSPv0emV3xaEwZj4q4wINxQwsQnPVjZ9/Z/IEMKLh8u+qBVYFZcAoofm29xCXQ9J9b98bWu63noCuai9WCGAC0oAPQRdxjn2mZ0CPhCylcf/gnqqOe330MDYTKBC3P9pQ/T3cN+INDj3mgzZjqH6yXTPNNnyBkYh/M2JODHHjpd1NN8KNPA90etUknc8QcMIHJxaHoesDdYj7/ooNhzY5/0vbiFRucNvwnCTl9KEXZtnrSjoZYfK18ozOXlZbHvOH0LWRMfFkfmt7uaRe0P22+oCdoR2IwgvmOwgDBCL5hOvuUFkrzXEh3AESEwCwd6qvXI0DP7WtFmftzUYrGIbDzHCvgaYZka6qWDZ3dSArbXMKk/uP4De6xYzEdySkJQaRu4cWliNRExXaGGJDepkaVQ61AjUQv57UgTATO6Tn686IkLYjRFsHVrUE4bB7iaizySsCNG+JaQ8YxJ3lHhTmBxBuALBntg2MBJwnjusxwhW5aZ6mXkt3d0PWCmhY3dKgHVDPU/gCbwXeaRSM14r3GtRmIgB74RAAC7C/of5KNboqawd/Oid5Pj502rL5oHYMYu4EdRd4b57V1+hZxe/4HXJgRdkgHMU71o6iNQSP65VDRgRPZt8CkBrpBd0ASHnygqHk8Ex/QuXLSgGOjW1VJMDq4ce5pHKbBE708Nw3yuofKnzPj0eZ0+wIB4PZwjSa34VlxKFOLiTVvr9GnAFRzngXD9Ir3XKNRPg64a6XBKUUDAgJptYcxFEuUGZ9C0Tb+LqBQ8pLVSKbYr7jffxE4QER6oY1wJiQf+0MJPz3LPcv8fbC3VecVCAQA="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514859.56/warc/CC-MAIN-20240718191743-20240718221743-00004.warc.gz", "offset": 110658803, "length": 7228, "data": "H4sIAAAAAAAAAO1d6XbbOLL+bZ2Td0A0E9O+sRZql2xqjrck7nFsT+zc3k4fHpCEJMQUyebipfvkAeZJbuc5/DT3LW4VQFKkRDlK7HT33I4TyyKWQqFQKFR9gKBvd9/s19Rq/UnpW3hXubj12ID4LPBcJ2Bx4gENIbFRb7Qq9W5F7V001EGzPmj3fogLvGGm61uVo4MB2Yl8ZxBF3Br0emp/1KiPKuaoRSst2u5V+qzVrzRbzOh06g3MHD4p7btOyJywcsyccTgZkGa93e7PkiVD1PNsbtKQu05tEobeNpkG4xCytDlWv6W+yZ2RO8cLNWidqWaj0m511EqrX+9VjLZqVRqjvqr2O6M6/BvGJKBlM/J90bibJdKyaN0c9RsVg/b6lRbrtSq0bpgV1uqaRoO2ILOXEDk6q+xaFjAXDIha71cbzVZV7XSqjVTQ1B+zsPL2zdGAYI+CQa1mcMPmbjhhP0esOuVBAN212MinjsmqI7/G4emm6k28f9hXtiaeuBPqAWPr3NKaKe0z3w1d07WBcmMhKbQDGO9m0lnuTZhfOY84DvHF8bm+/2oX/jfq+tnp8fcqjIZ+Do/tTkKI3toutSoHfMyCcECCCQVl2O1etL7rNLvnrZPDl43ji7evDhuvTi72v+/+8+LoRaIme7ZrXs7X/Gf75cle5+Wri28vdn84PHlxdPTDcXv37avvv2t9s7+XiNOC4eAjzqyUgUXFuJmEU/v5zdR+UnpSenVxcQb9VEFt6+RJyRIqfDGJtojaI99EtlBnkigyefn64knJjHUuFKRDdhPWkOQ2MSfUD1ioReGo0ntSuqL+7YDsmibzwsqhY7oWd8ZQnZoTVkEiPgrac4OwAinmpVbfIp7PMg/Lsp6U2I3HQW0G5JyGW6Sukm+og32ok3p9IP5LXoEdaMq95MDp2cR7fbuHylM5PdvdP9jTgBXWwbHLqtQ2iYlrFxFD2hWQQwXk0Cb1xqCdyGGbvKY3ld0x05p1+TPX2quz88Pz86MDzeh0qdrvt/pqy2Ssi3MJZpXR6Bm90WhkGdvEo+FEq83VnxqnHjUrx6cvj040i9ksZFaGNxwj5I06FbXfTfut5nmbZyomipyd7L4+1OKE5aWA/267U283e91Ou7jYwe7F7t7u+eFDmLQpDPUUNASV96MaGDD/ivkD4jaCax6ak8qZe838PRf15Qrm7HeVfZ9e2zBjE11lsfoNiOEvGtNGv9vt4HQo7Tw9ON2/+P7skKBKk7O3e8dH+6RcqdW+be7XagcXB+S7VxevjwnMmVrt8KRcIvBTRtMElun6+rp63ay6/rh28UZONFXFSsn7qhVa5WFpRxCHKegEWkFdUJa+rFHGQgObOmOtPPLL6QxTxBRTkBKj1rC0tjNlIRUWsgI6zK+0cnaWQkX5pJWXTdcyqaVkHDplWplG4cT1M1XPXu+Rl74beaJsvvAlu72GtS3IFI8tM7EYkbZ5i2Sn2WKDFgtMn3topDJk9uI662zs0yu2DVXJ6xnpF4J0AUe+a7hhlh9q23GbTyuVo0PCQhIADeaMmI+UTHcK85Ab3Obh/1YqRUJVvqu83YWVTxa0mZJSV44OtUNrzJS5Pl1xdu25fpjh45pbMNstdsVNVhEPW4Q7POTUrgQmtZmmbpEpveHTaJokgM6sra0B6ZCHNhvu05Da7hgkwRxi87FTIJKdmixbWsN/OzZ3LsFhsTUlCG9tFkwYCxUifANF6IQZBAqZ+GykKdWaLFMDkUxhxZB/qlDiH2q3Ue/0G61OXZGyfAjdEYikQq9Z4E7Z41N3PeZUAuoEj0/aF46cbvHAs+ntI9L3pkZLn1ojaMh2HYflee/2HoX2GGYC8znTvQl4O1+kieTNF6Htgzf3JQiHdPxFhHHFcWY6LApYMXmwRwQydLS46MBJR42ADco2ff7q9M3F/tsLcrR/epK0xqcU3MRaAH5pbUTBpMA8hRdhhqQ1lWxKu/+OXlGZWiaBb2pl7ph2BHY3k1OzfDrWHd3yXa/6LigPd2oy4wEkJ9SxbPaoJD3Xi+4npswRQ5kQssZHZOOp5ZrRFERdheDi0Gb4Nti73QcvJDgBu735K6nVwP2MfMIZWt6PlCcawVKjyDFx5DYcd6qbmLn5K6avgSdMYISmgeawa7Lr+/R2Y3M7zTLdCBaGukwYuT7ZEBUghfCd4qYv6Bgb3lD+S9ms2sKNgcLPn2+SX4ngZU10dIXKyNcG36yas85oZNYBInsA5JD/HwWrz5//BD1enfa25Ag9pffynfzjsxCiRikZ0XmRjr8g/ReuFCahJAphTQ44LNNiSGxFeHOg+LDaBeTtm+MA18HZSJdmQ0FEUaZDoQ0IbqgcEBQvsL8PDtA5uAGZvqBPRM2QifR/LEkf5NPBhxLc89FGlqhGym8vXoBnFStB3FvJEAS06Ea4DhCRjAkS75kdsHxx8AA8li0iRTTT+tInmqfCOe++c2sWf8fDGpigKWSGkQOGIP5TFQTAVn10gknJvnPBwx7xMUgWO+OhjE6dY4hHIV70IaqCRAhywbUZEGXkV0a+gkk8OGBGBD76iIIYMAVM5plNufMNeGqzqhB0XKLZG5AfhazQ1QJCYHD3aMAEKUEeFWCgVKs18V9Bwf2EmRbzsK4CUbEw07XXLAiQ4Hno+kwRZYA524B2BumkzhWSY3rNHcu9rgKJA+gz9va9GKM4oTqNq0AOTvwchY1fI98G7mr0HZVoBYQ/ETh7+LwOzLOxltRXtogFAZUZHoGfOBDSeS+0AbsE3EKrM324f4yERb1XC5IXMK4YYCR0H4MsyCWlvCrDQ1QpF4ZcDybutR64JjjKusNCiDcuCdjIle2/5LGIw4BPwcLoAaO+OZnjbiWixR3HyEH3MdgJwnmqIFgZupV2DNe6Ja6DcA1GB0KlLDaikR2ehzSMAk3B0EtAKXe/BcSLICIyle0yIRxqgFBRPOUVZmdqFUfQyokLngaLLfcGt4RKo7AjW3Mi294WdtikgbCyjigdELEqcOxSKbGjU0mpeEnYuz2yNhRmK8+59VzZn3DbUjYlaUky8aJLwoA+TcltrkZZEpEkhQFOqqeGPrIDQSDOWLJcRbakkVt/CS7AUD9eYYlcYqV9hsYg50f+ExLcDUOfG1EIhDj0D42/4jpjm4WBzgPD0uMxex6LOWUNOJNU5OK/ZviMXsr371NrT9JVASvZ/D55yFZFo6K1ZGV6avNNsnL1DLsJgaS+4NnmVTDqUOsE1rLteNGOV6wINed9RtnEvL2e0HDj58jlEOxa0quQ2I1IlEIBXWNEOfzu7Fg/Pt1XyEDIYSmvFr/S2Y1n62DrFfIc6VbFAliNVQoYVQwEU5Xt1SgtpeLAQq1sC3t7D5lYeFlKGa+KKBlN0LkD7gS/+jSq2Z4upZzQLeX1aSbcxxGsFMkD5brS6HyKWHOdfzSRzgZrTqixmY4FipqsEXythu4xQpL7IPSspy8mT6Hx3S4lPntkp053GjcQYSeWmC6bx6arwHiJ2kXmS8xp6dN/zITpynPsU8Z2JdEA1FxtJD6mJLMW7teRnIc814cZJ09nrGNMr5D1dXJ/MYicVykWROPxCsUMGlyCBz6T1griytiDtNYykd0zbFWfgdxMlui4soXyVTbvn8DJajP7+34tNkvv8xr/Hk37ozhHIYV+ByB6dH8/xSFcgeij0ZvywFzwB0Gg6HgpNMTKKFZN+aVyhPuMA7Ver2NpKJMpKsB4CGOYXx6maVNImD0q+KijS8h8IDBpDhPg++43hLwtRmxahPJCSdlcjrA+cf1AH7luiG3GY/sAQSA0pSeYS0YcQDNpluOGnh6HLHHHhLqDi9ooJ0zmWFUcmHhjCuEQRhoB1HfGw10CrkXIJSbPwpDBM4T6V24UYPx/xX4hA2hfFi7t4JAzIkF1BcT/DLgLffi14tZlI8IeuQ7IhPugdcA+Y45ChjtUBsblatG+cTlPA3epYkcYWgk86gzfsNAFr4f5hEa4RTlFmOLugwmjQ+gVMwnE49AT3+F3v/kzP7parUIfkMBOjcJvaGGohXwLHWaJoBAaHDF/iq6VUIK/pSJDfDCVvgK0wVboQBEYUBK22dQLb5Ussb8fQQw+pgb1eVjFfaCtK6JWm9VGtUkadbWDRxfqXVLvDtq9QbtL3nnQOnUcRg5vPPJ30SgOFWh9FrMzgwDUrApqkvjzuhwYsIFxAsTvmLJdel/KF/Td68ViFUhdLGoy2y4oi8lgnTxq4R5fxefjSTggbe8GKexIQBbZTlRVkGQwI+1ATA6cbnI4D+4+YLKYbopQAYEipGMFBTPTOkNHz0xxpJgtEjCWKFL6LOZkrLxxlnjQcwWghJ+QgRnNw9u5dkQRK5ng2Vbj4rnRyHGRzxmmq0524iaHJ2x2xexGptCaFBo8q909mQrx6ZuLKjl7dXpxek4q5HUk4kTRTym3T//J8jKvMjFvY5QIHjGAoVzLtDF7m1jjxZxVmkmo67g/6AbMWrWZ4p9Svlz2QVqB+J0vFSS2BthMrmi6BmU0EFc+sQG5ltHoXLeyBfWQhz6M+2l05QuMSq4wRm7VYQ6ZTQICQ52dCDnbY4cUyJmThItc4+BoU8fate1yam+V2eIySLOFM7Chbm5LqCeLyujUtjfiDOG3X7k8gHUulwvN8uk4MX7wVvfsKMDf2eImN0xkk1itOuYjhRiuD0ZaU+qK6JHMhlwlts+y17k+wYPJ3oiunwsmysN1xwi87aLSpmvb1AvYUhlkCmxsLvZj6nInEC/zPUkqLu1LUuChvflc1yFgNjMX8C+pxvlHEe+gCuhjdLZdPxT7JVkYDYOppHQKMCSl9XiPfcNitkhArCGzhRHHYD4GYfcDr1h5PYgMLSG97tNfJEfK8w2UgTtKW9kUvn+EZ9DALlsK7hvEWWRAFGXzuQK0PM/XgZtMfcHbQlVITWuh46MzB/oV+YEGXsmz5kH2PNuzRodbkNZs1Gee/BrJy7CaiA46vlXfUpQtdSuGUmcig4z6ZiYYWJDxQo2N5EhhVr5Lg7yRDzGPHiN+4CbhuQjcIALHwsfzNFqeZXyDOrYx40mMHYZz98FXS5opjptj3U2aIPJHBNHpBhzfEbvC8Q5bNnxeS+F/dkXtDSyGIRlyPUeRpMHV7E0q10tuS1XRBe8b89oqsrAb2id2eiY4BBVSMpsz9tO0DJYHyjJ1r5jAaTOVFsLExZmcMSYQkZ6LSe/6wti5zj6EuJeaIpQIFlbmw8AuTlsw4ITaoVa+8DmsRxz1C4/ZBGUizrIUZcTxlwkTBFxiPE4Ic4P5yVKQt5XCMBq3Ov1FWkqKZ2c0xXDD0J0qZCKMoKY0F2yoYBM31/MWFDRlKg/6KFNTBzWAeQZUxchKc5iNKKauz3S5VgZp4Jh4siIaJ+DnT1xoD08+YkjEHS9KbO6EWxaGLLI9ue79+JNCRLOQoLeU2vCeGkIuel3Hgkml+2u43lzxw3/dX2HEmW3JOhnO6KAx+DUYqN3B+s+RG24HzOcg+V+YFa/eMnk7GDTUuAgdqFilEz+eH+6+2X8VFzvZfp9WUNWUpnADkKkZtXo+E/wJDxxswWQQl3qv1O4VM1rgtCOqKFzDUV919VzuEdARmnxfR6eJQQ2W8QuympPpmY64kqWBhWGQnUpvihvcQE6DX12uZUo8YZRd2Qp6dmlDyrAoNavXAnJDAWHUlFmol+8s4Q+4qrEVChZ9tx/vzY0N7E9ap9vobpcWLEwp6/SWmY3FzoTRSkMaSbsiTVkSPWUFnjRuMc/mVFqPxDLFHipuH+NmFZJXtsR+M9ipRWd0QxEllrqjaT6J90TElu12WZxAwSWmyE2VTlS5WgO7CNZJDL7jTuGthtnrrjgZ6TBbU8vx6UI+PbwpZ0RyNB2nptKSkWw5MWXlejkxcOUmHoRM7WM5tY9o4XSqq50btYMWsizMsXIQrzzysDcjMI98Pl1n1IxCtp3q2keK1ZaOSQVhL1omePJojCODGfJRQgyymOypPJ4ECTr2uJyfVAHulyJEAyLK+7kx8KLJkSGJ8CXgposeKMN9F+YOrCR4NhTcZonYWBH0Z+oFBZM4qS7DbXBwSY28jG7J3t2/IRYuqvBDs9/rpRIzwxv9CsYGUqu9XqVRr7fW6dTb9kchrib6aBpq+DGNZ81dd8SfNV5AAry/ZFfwOg1v4NVw3cukDngGjs+0XJIhm0p69qxRZyG8QO+e7Tef7fagh/BoRfAieplW/Ox6aDIDrdHtQ5aXJuMnCzS13+2sY+dgINMcD9zsczYGKzQjIjBsDWY4n6XRCOE3bQ/b7/VBvpkcAeppIHolHfXcn3TeGT5J9HAWTs8ZFuH8zNsVUybKVbs8pf6YOxXpOAw63s12dh3HnfmcvcClg4TuJQOvoGvUe1a7x3r9dt+wDErbvZFKqdluN61+v9FQCIoq5Lj6qN2G2mz2Gu0egppLgU4BOEW28FQKdp2lxscKWLQrLQ/GreFhwwyNotoFGzyzVetviV6Xz0QJNK+mNK+zLWBFHqkH67rMRsrKcjmyeTxCec6QjaV8pTspRZwdne8dFPOFOfdwhdl5nkrSwV+TOE1kS8BmhiQpvns92wrIYDa44ZWV7ZwnKBbf7WTAcYwTgmFqqBDdjp1XAXamgjDGlbHPbvP4iwLq9LNuTmC+KMMLhH0Egi7nB6gPqlCufCzJVayidHAG5Bj9y7sPY5/a8lie5979NmWy0oQ6AYSKuTYF2J3v260HMc7DOgfLBQJZSayUcDff2+wCdfdhGUNg1rHfD+JoV9IgizwUensYJ8jVJPm8WqvV6itkmF1W6HCLSLrLOGeW+iCuDy2O5AvUZBnTQmUCMOIJ32obwqihMOHQ+djAp5sdBTzfgql8mKgd5+6DGH6pvwK0XKYBuBQt48QRS9iDeDma4qc+cIeuYORhbSTessYDCIwe1PQLCE9oWNRsm7SJOV3WMH7KKHpgt48ljUW9wd3Ku/+hoAsfretCo2DD7QwZklZfxryA7R7E+j5F8wWNs2CZzixVftF6OmHRwRvuv9o9EVNVuhyrVW20WjhnTg/Pjw4zdZf0OUECH6aoghsC0YIJQYDNPrnvBZ+vhT6IrQI6lKM9LNwYWlwN4s2OUrJmCo8tu2rO1v0C9GT7EVcq8mN+hfgJHPvHsNeSh6r8U4nBgc8xqUlnMnZ1iwj3uqAFYW/k220SW4FqrFsfmY3xICXNfdY0Xkaj0MH5OpH//0zk5MjHwgQXcFFptkeJ81tsR+rsBqaqTfHTw+CFzB5yG/DJtjmz7XjLX1MaszMPGRL5cKkk46XJrKRnx0duJIhjGmhDLFYxqC8aDSf3l3dD4R+H7L6iIXijus0NYBdKn0ceugb3VUg+uzCrdIwfluBBfBZhec2AmfmK5zIhqTM8iLFz/JCrcHwnCwrJrpijZM/PpKLp9Bpqokf5TCEHoSr53HzXj/mVz+aLFHSWkTMG8o9uSAA+6GvqOwu1Fjr6mo5pwJ1C9gIeRqINUCkw6JEj9CcxgPIg0UG6qZCeLSIzzS1lVTddkgr2oXNR6WJ+YTyYR0NzFHJZ6TmeDHLxe+GyaquvNlYAZkW5L4jMCvr3Q7OyyD3YbFLgdwRnRZN/LXRWdPnT4Vk5OEvw2ZfM9fGoyp4PmcxZFZLdtb0J7lSTvdOTk90Xd//+MyGzcZ+eNepJr+Yg2PsKSKxV7ddXxFpx6w0qnLkCxO0HnKXAbuz5fjYsK0QrsdkFYDYZgIeis0I5viw8O2KqUe90rGZ7ZDWo2e8iVNvrjNSOOep0O9bjw7Oxxn8uPpuv/qgA7TJDuSJCu5yzB0K0y/j6Ihht3Is/O0i7zDR+xVo/AWvtt5sYYResF18h1y8LueIhxYdF/UBAxgSfMvzYbBqzq20Ebs7cuw+wLub3LWg2GPh9wWJY3P8IsFjtfIWKvzjC1GghwrR3dPryze7Zq78kVJTrnQf2yhXnsx7UvxN3Sjzf9fylPUtWSRIvm58KQhc7BTEKPb8UPwBMXmVB+mMwZWGWEkxZGIt8/sYj2NxN8hWl/mpD/nNtyFf8+/8X/q22673+VwD8XgA8H/R/FgKeJ/HngcCbare3CgSO5b4kBI70PwKBiyL3QeBxgd8TAscm/2IQOHb5MyBwMThLIPAzWKSA9QkPQpf7yb0AU7k04uc1DyNYtdiKyPiJ6xvMD8nB2xenb9/s/+vPBIzLnj5r1JO+4nljfLEpvMQ9RvjagRfZ6znk/EEUYmi90yyC1luNwmPMx9R3I3AKVgHME5EvgOXxmDwYK0ct+rJYudGrt1v1ZrffMc1um43qqkUNyzJos1mnhtr5Ali5nBqfjZXnqj8uVr7Eoq6KlS/l7KFY+RK+vgxWLnvxZ8fKP9GGfoXQPwFC77WabYjjFheW/wgEvd9uZxD0xJz/mSD0VuMPA6I7zT/k1HL9KxD9xUGkbgv1/vXb86N/vf0rIkirgb6Fq1sM+q64pjwAC17Bsj4MCs4Yv/SsdMYCIhYMfm8RFoyWIT1fXP96uvjrlPt6RvivipE2m83WV4z0fow0F+x9HkaaI3EPRrpTy17mkL140RCO60T0z2R4OcjHbxDHKwQzNzuHLAh1vA5jA182S2u/4nUReN8MPlfRIa2KuzPIkKh484y42wYMjx9ulE8iWP98V3i+eMEjyNMjdx9sdnX3gTwtp7dxx3RGECMF8X1AuVi2FN9XVNTszorNGjT45CbTNARixZXX4kLG9L78zJ1yUtp4Q3vmxhh8u/yymAJLC9Ya5j1+21OgNdfxlk9t3Y7DUI3OXR5DXCeIjCkP8Xo2yWZ+sMT1Mgs35AgISiwjyQ05IV7r4sfX0SyUx+uG5ouLK4jAbYAaQsJqTS2q6oB2zVfFtOKGEDObL41pSnojS+K87dRibZ7fKpDv1iRmnxZO0rIXbnKHHBz9N5m/SBUvv0wRtvRu1VK6PsVpui8uw8Qr/iVkt1AAv7BDDyJPXhW5/AbSkStiOgERinfoN5SHr/FLgvBTaja4ItIHgtVXLuZpo0tavWYGflkQtgwNx/oZF5B3/+monuWYp/jL6T7ylZsppr/4xWzDgitrF5hdW8atNzVw1sQSynxT3th1xzY2TULxDaGabkAseDl8gWs6iRw2u9hGXsP6UlTItr2WEYDNmYONJTfm5nuPDQZ8bFQdFqZdrVar2Ezg2pGwhbDIQMvyC2rA8b/7YHEaX6MIRcsJn4pkVBlCa4KbVE0EAhcrJCnlNbIWax1qpLyCNYP1iot3y8P5m3jFmMX3jIJHVHx18BzGV/bU8vzik+TgYX6JzJanzIkOGIQG0JWwPHxSImTH5oV36ZaHu6YZMW6nYONC6b+VEXbCj2rmLp8UWv1EfL8itpvsn4BhAf2PykmWpBRLtyylmx2/hW+MResAjTFwM4Kq6U7Lw32RQs7vfrvyWZBjdMUG8As4qtz08Otn8X2tasGIHDkBKEsUEggaJq7N43ucRYT1Oa0YUTUy7duVGjm+RY8u3waCukL6c8OwvNVY+QtmfPG8Lh7gGn6TLX71bEYrXsqkTCXB3ry1Lp4GQsHF96OtZSeCQYE+jfT526azVdPrc8U0kvXvd3rwuxl13NQDT2CtVkvtiv/0abrYE8EnfpGL5BgvVsZ3+J2i/weMGLU7O3oAAA=="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763861452.88/warc/CC-MAIN-20240725175545-20240725205545-00033.warc.gz", "offset": 3636501, "length": 7712, "data": "H4sIAAAAAAAAAO1d+XbbttL/2zon74Do3oT2F2uh9sXSPV6UxL3eru2kbXp6eCASkhBTpMrFS3vyMvefr3mOPM33Ft8MQFKkRCmKlzZt7DYyiWUwGAwGMz9Axvfbp7sFNV98kvkennLnNxPWIg5zJ7blsiBxj3qQWCqWKrliPVeqnqvNVrHSKtXeBQVOmW47Rm5/r0W2fMdq+T43WqW6XlSrDZajzX4lVxmUmrkGZTRXbRT7+qBe69fqg+6TzK5teczycgfMGnqjFinXG83mNFkyRCcTk+vU47ZVGHnepE3G7tCDrM4Mq99TR+fWwJ7lpVJmNUNXc7WaXs9VGsVGrqlX9VyjWKvWS2VWNCr1bkACWtZ9xxGN23EiDb1KK4OikdMbJSBSrUP9Un0AAqlWjGKpadQHekhk/yS3bRjAnNsiarGZL5UrebVWy5ciQVNnyLzcm9P9FsEetQqFPu+b3PZG7Bef5cfcdaG3Bhs41NJZfuAUOLxd5yejyb/MS7NDfW9kO5rL2HNudMowOgHlE8f2bN02JV0YWzXMoDemTY3cHh8y12sRd0TV1t7LtwdvD093e6/eviy9OXn5em/3sFz98cd37456hyeVWlB3x7T1i2TNndc75+/OT87fbPd+3K7svn231zv7z+vv3x4cnf2w8125F0rCAEnyAWdGxMD8mF6PvLH54npsPsk8ybw+Pz9BtkHjiuT4308yUv/OR/4mKVXJd74pdJEEWlglrw7PZxXGY9deAYm2iT6ijsu8ju8Nco0nmR9yuw69MpmTOwfJugN46Fm6bXBr2IKyvnXBDEHNYjry1iIXjE1y1OSXoGRvqXPTItu6ziZeVA+KU33EUHM8BwU/sV0vByn6Rae4SSYOi70synqS6V1POGhMi5xRb5MUVfIdtVAGRVIstsT/sqdnoDe7tn3BoZ8no8nhzQ4qTu74ZHt3b6ejU53VysWqFlenNmGSeOfcZ0C7kQMp5kCKVaRdCqXYJof0Orc9ZJ1yUf7MtPb65Kx3dra/1zFqxSKr1UH9Kw21WqGlQaPUrKjGoFkzaiW13yYT6o06hZn64/7xhOq5g+NX+0cdg5nMY0aMNxzhopqDfufUZj3qt5rkbZapgChydrR92OsECYtLAf+lYrWkluulUi292N72+fbO9lnvLkweUBjqQ9AQVP7P6u8Zcy6Z0yJ2yb3inj7KndhXzNmxUV8uy3G9DTV9qrZ9J82OQhdxOmW2nu4d757/eNIjOCHIyZudg/1dks0VCt+XdwuFvfM98sPr88MDAnOuUOgdZTMEfrKBVbq6uspflfO2Myycn8qJqqpYKXzOG56R7Wa2BHGYwpbbSamrNptNWSOLhVomtYad7MDJRvNTERNUQUqMGt3M2taYeVQYsRzoML/sZPWgj2j5oaJ862QXTfYsKURkLDpmnaw0mrGqJ4c75JVj+xNRNln4gt1cwbLmxooHVpkYjEi7vEni02y+QYO5usMnaEhiZHaCOs/Z0KGXrA1VyeGU9EtBOoUjx+7bXpwfappBm09zuf0eYR5xgQazwKwhJd0ewzzkfW5y7/9yuTShKj/k3myDTsmCJlMi6sp+r9MzhkyZ6dMlZ1cT2/FifFxxA2a7wS65znLiZZNwi3ucmjlXpybrqJtkTK/52B+HCaAza2trQNrjnsm6u9Sjpj0ESTCLmHxopYhkqyDLZtbwvy2TWxfgq5gdxfVuTOaOGPMUItwCReiE7roKGTls0FHyBVmmACIZw4ojf+WhxL/UeqlYa5YqtaIiZXkXugMQSY5eMdces/unbk+YlXNh2bp/0o7w4TSDuxOT3twj/cm4X9HGxgAaMnFVTfJeb9wL7SHMBOZwpk1G4P08SBPhw4PQdrjlPQRhjw4fRBiXHGemxXyXpZMHe0QgQ0OLiw6gdPQI2KB402evj0/Pd9+ck/3d46OwNT6m4GYWXO6xwoCCSYF5Ch/CDElrKtmUdv89vaQyNUtcR+9kuaWbPtjdWE7BcOhQszTDsSf59262u1WQGXcgOaKWYbJ7JTmxJ/5yYsoMMZQJIWt8QNafGrbuj0HUeYgreibDR3fnZtekrnsEdnvjN1IogPvpO4QztLyfKU86BEsNfEu4weuWPdZ0zNz4DdPXLikQ8tjY7Vjsimw7Dr1Z32hHWbrtw8JQlAkD2yHrogKkEL6V3vQ5HWLD68r/KBt5U7gxUPjFiw3yGxG8rImOrlAZ+VrnG3l92pkOmXaAyB4AOeT/J8Hqixc/Q49Xp92WHKGn9EE+yV8O8yBglJIRnRfp+A+k/9KWwiSU+B6syS6HZVoMiakw9OZA8WG1c8mb0wMX18HpSGemQ0FEUaZBoXUDVk05ICheYH8XHCBwaON9QZ+I6h4T6f9akN5KpoMPJbjng/U40Q7Jvjl/CZ5VoARBbyVDEMuiG2FbQEQyJkh8YKbLksXBA5iweBEpoqnWZ77QPKXOefu9XTD4e+4VwASNIdPzLTAEwa+8IAC26rMTTEr2vQ0e9oAPQbLYmQnK6Ng6gHgWok0HoipIhCAZXJsWUQZObuAomMTdPdb3wUcfUBADpoDJPDEpt75zMbgMq0LQcYFmr0V+ErJCVwsIgcHdoS4TpAR5EZIq+XxB/K+g4H7GTINNsK4CUbUw04VD5rpI8MyzHaaIMsCc2Yd2WtGkThSSY3rFLcO+ygOJPegz9vaDGKMgIT8OqkAOTvwEhfXffMcE7gr0PZVIxdg2fHD28P05MM+GnbC+skkMCKh0bx/8xJaQzgehDdgl4BZanerD8jESFnWpFoQfYFwxwAjp3gdZkEtEeVWGu6hSNgy55o7sK821dXCUNYt5EG9cELCRK9t/yWMahy4fg4XRXEYdfTTD3UpE0zuOkYPmYLDjerNUQbAydMts9W3jhtgWwj0YHQiVMtiA+qZ35lHPdzsKhl4CSvn0u0smPkREutLOEsKhBggVxZNdYXZGVnEArRzZ4GmwwHKvc0OoNArbNzuWb5ptYYd16gora4nSLhGrAscuZUI7OpaU0peEnZt9Y11hpvKCGy+U3RE3DWVDkpYkQy86Iwzo04jcxmqUJRFJUhjgsHpk6H3TFQSCjAXLlW9KGon1l+ACDPWDFZbIJVbaZ2gMcn7iPyPBbc9zeN/3gBCH/qHxV2xraDLP1bjbN7RgzF4EYo5YA84kFbn4r/UdRi/k84fI2pNoVcBKJl8mD9mqaFS0Fq5MT02+QVauHmM3JBDWFzybPA9GHWodwVrWDhbtYMXyUXM+xJRNzNurEfXWf/FtDsGuIb0Kid2IRCkU0DVGlN4PJwfawfGuQlpCDgt5Nfilxq4npga2XiEvkG5eLID5QKWAUaWPYKzSXo3SQioWLNRKW9jbJWQC4cUpxbwqosQ0QeMWuBP88suoxnu6kHJIN5PUp6lw70ewUiR3lOtKo/MlYk10/t5EOh2sGaEGZjoQKGpyh+Bn3rMPEJLcBaHHPX0xeVKNbzsT+uy+GTndUdxAhJ1YYLpMHpiuFOMlaqeZLzGnpU//OROmKS+wTzHbFUYDUHO1kfickkxbWK4jCQ95pg9TTp5OWceYXiHPn5PlxSByXqWY6w+HKxTrU/cCPPCptFYQV8weRLUWiWzJsOUdBnLTWajjyibKV9lYPoHD1Wb6+8NaYJY+JDX+A5r2e3GOPAr9dkH06P5+iUO4AtF7ozfmrj7nD4JA0fFSqIeVUawd5dfcPu44ttRisYiloUysqADjIYxhTrYbpY0hYfqq4KuGLiFzgMCo3A2B70+/I+RtMGLSNJQXSsrmEoS1ke242sC2PWwzGNs7CAKhKS3EXGLiAJphsxw39LQgZAk6JtQdXNRSNmQywapiwcQbUgiHMNJwob417G4TcC08LjF55nkM3iHUv7R9F+P/S/YraUH7snBmC4ecEQmqKyD+Z8Cd58A/I2hdNiLskW2BTLgDWgfsM2YppLtFZWCczc/sGYu3bJIGPHuBIwytuBNqdU+ZZ4PXwxxCfdyiHCNM8emjDqND6CXTCcTj0BPH4p9+d6Z+dD6fhz4gga0ChX+egaEW8i10mIWCQmhwwJwxulZCCf4RiQzxwUj6CtAGW6EBRWBACdlm44l3o8SJ/XMfYvAh7VOHe3ncB9q8JGq+nC/ly6RUVGt4aqFYJ8V6q9poVevk/QRap5bFSO96Qv4pGsWhAq2PY3a664Ka5UFNQn9ekwMDNjBIgPgdU9qZD5lkQce+mi+Wg9T5ojozzZSymAzWaUIN3OPLOXw48lqkOrlGClsSkEW2Q1UVJBnMSNMVkwOnmxzOvU8fMVlMNwWKMd+JxgkKxaZ0jIYWm95ILV7EZSxUouhdzMdAcYMs8aIlCkAJJyQDs5l7NzPtiCJGOLnjrQbFEyOR4CKZ041WnPiklXRMdsnMUqzImhQXvG8L8aytnTEH5ufO8ZuDV9v/Pn5JwqKBxOI1V/6ZYSShK2JmwjijOPBsAYxhvJHpY2iG53NWaSakruHGoO0yY9VmZn4yyQLxFznvgydHqkUw/5F+omi06sT0Dtc6seW4FtPhRH/iBTWPew6M9l7gVoBd4nggCHcyXWGzOKRNEJBmHklR/oSxMT0K1PRRyESibfCsqWVsm2Y2MrDKdDVpRdli9V9XN9oS24nDMBo1zfUgQzjqlzZ3YWFL5EKzfDwMrR08ahPTd/HfdDWTOySySayWH/KBQvq2A1a5oxQV0SOZDblKYJBltxN9ghednYqunwkmst3nVt+dtNNK67Zp0onLFsogVmB9Y74fY5tbrviY7UlYcWFfwgJ37c1tfQWXmUyfA7ykFidfRYCDKqAN0bu2HU9skMRxM4yewtIRohCW1oJN9XWDmSIBwYXYnkUQdDkYdS1HWrHyc9fvd0LSzx36q+RIebGOMrAHUSsbwtn38cAZGGNDwY2CIIu0iKJsvFCA1mTiaMBNrL7gba4qpEa10NPRmAX98h23A27Is/Le9PDas1KNG5CCB9imnvsaSYowH0oO+r1Z3FSUTXUzgE6nEoOM4kbM+Z8T8VyN9fD0YFy8C4O6gQMxjhYgfOAW4TkI3BACR8LB8zOdJMv4gCq2PuVJDB2Gb8vgqgXNpMfJgeqGTRD5I4LmaMONb4ld4GBHLR4ur0VwP7uk5joWwxAMuZ6hSKJgavoQyfWCm1JTNMH7+qyyiizsRucLOz0VHIIIEZmNKftRWgy7A2UZ25dM4LKxSnNh4fxEjtkSiEDPxJy3HWHrbGsXQtqLjiKUCNZT5sDAzs9asN+Eml4ne+7wxGKUJeLsSlpGEG/pMD/ABcbjgxB3MCdcCZKmUtjF/o1Gf5WGkuJZmY7Stz3PHitkJGxgRynPmVDBJm6mJw0oaMpYHuxRxroGagDzDKiKkZXWMB5BjG2HaXKpdKNAMfRcRfRNwK8f2dAennTEEIhbEz80uSNuGBiiyPbksvfTzwoRzUKCVlEK3SU1hFy0ooYFw0rLa9iTmeK9/yyvMODMNGSdGGe0VWr95rbUeuv5L77ttV3mcJD8r8wIFm+Z3HZbJTUoQlsqVqkFr2e97dPd10Gxo/aHqIKqRjSFF4BMTakVk5ngTkzAqRZMukGpD0phqZjRAEcdUUXhAo76qovnYoeADtDiOxr6TAxqsJhbENecWM80xJGMDlgYBtmR9Ma4oQ3kOvBPk0uZEkwYZVu2glhB1JDSTUuN67WA2FBAGCXF1unFO0n4A45qYIXcedftp6W5gYH9uVOp1RrtzJyFycRd3iwzsdiJMFpRGCNp56QpCyOmuMDDxg02MTmV1iO0TIGDitvFuDmF5JVNsb8MdmreF11XRImF3miUT4I9ELFF286KEye4xKR5qdKHyuYLYBfBOonBt+wxPHYw+7ktTkJazOyo2eA0IR/3rrMxkeyPh5GpNGTkmg1NWbaYDQ1ctowHHyP7mI3sI1o4jWpq7VqtoYXMCnOshAGCPBrOCMwjh4+fM6pDTNCOdO0zxQoLxySHMBfNEjxpNMSRwQz5KiEFWUz2VB5HggQNe5xNTioX90cRkgERJd3cAGjpyJEhofAlwKaJHijdA+XTf8eMOL7rspQ5G5aWHhi4s6RAjkBmkEl2eqd7+9u9ly/T6r0rNxuNSE66d61dwohAar7RyJWKxcpzOp60nYGHa4g2GHsd/B7Gs/K2PeDPSi8hAZ4v2CV8jr1r+Ozb9kVYB/wBy2GdRFJfNnXwrFR/tlt+tl0ag7NYFP2KyqxSBM2f2ymVVciaRMkGuMgdtVmqP0eWYVCiHDTtUGGXjjhzXKhkIFGTwoeFUJ1p4vt7RMhEo81pWxNwtndM2zegAPPg4xW9mWYiht2BGc/dKI36CL91IsHHMgSm1wmGZq5CCEzMVRDAxWzxd3vv9nu7r4+/P/v3/lyNQ+CICqcgCIvjv6LJ3ndIqPzTCH7GmgmPa9aY6TJRugrZMXWG3MpJb6VVm1y3484Dbv8njBSuV8SzLxi4IrRRHDT1Eq1RQ61VDcpqVbVSaRSLTaOi6hVweHBMPY5Lnlovqc1ypVSrInK6EE0VyJZvCvcoZWtbTrNA/9O2vuXpuzU80RijkVY7ZRdpulT+I5xW2RNRAm26Lm36dJ9Zkef2waQvMsyyslwDTR6MUJIzZGMhX9F2TRpn+2c7e+l8Yc4SrjA7yVNGRhVrEhryTYkRTVErxbGvpvsNMZgId9Xisp1xP8WK3w4HHMc4JOhF1hEh9MBjFohqJIj+MDd02E0S81FAnX7R9BFMGaV7jkiTgOnl/AD1QRVKlA8kucQUi2oCFE+ydzOB2Ohu/MEygxhvGGNJt61FZhmOL2yfPi5iSEJl7p04kmAqIgFzQkvzElO+1aY2Yf53U9Yn2t0kkjxpk1WpIcRAujPQ7q1pgcHpSttJ4vY1Tm+RcJmh3kmwPYMLdueVcRH3QjFd8JLDDpRQFGIxguERaxZ+eQTWq2jvJoXvGzDKd9MIy/r0UWipnCniOOMiRcXVeREniAfeiZNdICC/4Pcl6onNhhKslKogwsBNCPYyQweBCPcAuorSJJY/zpMGWdQXS3god+rN/hi/kIObpym9AdeHTBY17kIMe6emX0IkSb20ZkukSvTxoobxC2D+Hbt9IGnMzwPcSP70vxSU+7N1bWgUVj4zRoZE1ReqHwKsd9M/6n36OLTR1Vyk/4u1EFuP1BDt0Mnr/YPjs2P41RPzV3hri5gP9pq8u2mc4IhAhKZD4GWyL+5EyEVkkIRxrtADMWXkzmXvh/2z897R+f72wf7ZYY/kyOHxXu/0KHjpXeM2jYXfMsMHV6607meW2glMV1sAKHcSwJE9JsMwMLSHDp2M8AjpIjmcAmucxXgK9sQyoZ8jvOy4pzP11VJgtnbSuyA/JVf1nyGq+0bWWCnwvPyVC7CqWyyD4cDNroWbRESJKa0Iuyof2ySwdskS6/e7kLRJY4Pkg8n9GbsW6F/Yq1sZxEU0Uh3sR5P4aBLvxySGp6rmTKVAaDPTQwFoKcX+v8au2RhsI35BH7zj6Utyqz84nQLzKjhV01FK02NFMRJJsCAj0YLRtOTEDE61SdxU76M/abBcnzqiUW+0vLyNhwt24XNZUQ8COc3kfTQDSvfMn6CLt6xC+PWgaaUD/D4Sd0Xqspqu9IRjrcmEsE53L9iuwu+Ri5hxNKdG7JJZSvyIWiSaSr1aDAc9mSnkIPQ7mZvs+gG/dNhskZTOMnLCQP7+NXFhmYFlw5qrNdfRQzqkLrdS2XO554s2QKVgXfUtoT+h+ZVn9faifbzo+F7Mz8/EVTda3FNOfiQwmfn8VDQkuQGRoJDIio7KxXC7P2orpKaWaitshWCxB9wKQfLLt0JEiSVbIUH+H7gVgi1+W1sh2OMv3woRI7NwKwQs5sg27Gu+6lbInOP6dW2ETPuTsgWSlhlsftRTNz/KpUWbHwcMNz4cBn4AHjBObHC8zEPWtqlTa6WtjeU7FXfddsDhf9htB1YsN4xmUe/XGrSv1in8lI1aqVlvskq1qdL733aQKn3bbYdE7XvddlhgBFfcdljI1x23HRZw9SDbDrIPX/+2w7zZe9x2+JJth8+CGF8xsK8W6zFk/2WeCFP9NaH65dJXjOo3K7C2Qyzhkmj1k9H3nwXe1/8U8F5tPEL3D45TleoIVPVeHeyf9c7I8en56+O94x96Z39ZvKqKSHK9F4Orjnu7bw57ITp1DDKDKBoi9OeWQd1RG1LkSpUGUa2EkKcuyRFCPl0I74KQf3Y5uBsGHbPYEdQaM9sIQIOnngZA12MAtJixt4GfF1i8R6T5cQbf0wx+BHT/XoBuDaLcJYAuqs83jucmgt1b4bkJCl8PnltsroTnQrGHxHOB/GfwXCyxDM+V+X8kngstfmN4LvT4FngujswCPFfNkwNKxF+PcyHc5JfcYrhTb1LijT59pJY3cuzJ3wLrxdPmQUfxBLroanioXBxF90by8HnU6VlA+C4UJGpcbpTSUONK+Tao8bbf58y5B8R4tvj20d7p8dE+lJ+vsAtcQGDt8Tsfb0e1fGCcuW8wg9Wa1UaJldRqkzaK1VK1WVerapOqg8oD4Mxiqt0aZ47Xvl+cOd04r4ozL+LrrjhzOlcPgzOLPnz1OPMqtjgMSfZ88pY5fVh2LJ3iiaZHZPpBkOkvOF7XrCKtqY0kMVMqSZ471PD1rxzwLpdjeLdcZr4mtLtS/orR7goGGSlotzixrv5poDd4Hn/KifXyXxT0PpXfMPrqAe8aokvnr3vHeAFQb4qXPcJlXwB4p/kGIeB9pxX5QSHyB1qX7oa8T5eOUCjT9QNhdwh1UugL6xSd+y7f+tR3qultE/XPRN6lJfnKUfdHM/KIun+DqLvaeETdl6Pu8dD/dqh7nMLXg7qrxfpKp6iL9Qc9RV2sf+4UNZRYeopa5P+hp6iL9W8MdYce3+YUNZ5BSEfdS4+o+9eAute+RdQd1PJhUXe9YlQrDaM2KFf7zXqDGX21YVRrg2Kp2q9U1epDnO7GqXZr1D1e+55Pd6ca55VPdy/g686nu1O5eqDT3diHvxXqfoB/aN6hOgzTI97+iLd/E3h77a+Kt5e/Obz9r/sXYv4qiLtawfOpZyen++fkbHv/6DyGkT1ibX/YGfUUx+I2kH1sOX8E6+fA+trnwfq7/JGWBXB9+RGuf7RBX4MNesT7/2Z4f6m47M+mPOL9WgJ0uOUp+xiFJXj/ViH+5+TjV731RXQzEr3TGV5P8Pk7i/HSsthdsh5zPQ3/IP86fmxk1n7DP1iPN17gex59/7z46/2kS1S8+0LcrgGGyPHWs0c+LP6OLcIjvFIOpDkhnz6a7PLTR/I0G93/G9AZQGzuBjeSJDCVTHBjSlqzWys226fuFzcZpeG+grhkV1wBF93QHbvMSkob74SO3VmBj4uvq5h3fMB4w5SHdYC5ncpzM8A9OnTm3goC67vfH3MPL4aS/CVHSdxsMXc5hwBAxXoSXs7h4Y0STnATxlx5vOlktri4/cT28aJyIVq1oKZVtUCtZqtiWnpDCNrOlsY0JboMInRhtwqBGkfv0RwST2ty6ymRO73YS17uB27g3v5bMntpI160F6G70T2OmWhVCtI0R1y8h9eJS7h4roDJrQvN9SfyarrFtx0ObBE4C4RaPAGNerZ7iH8gDQaYmODgSK8KFmK5rEeNLmj1ivXBwon75aDhQDODAn1xu6WGipkNeMJrn1qFQj+49xJvMcqP5YWXBhuI6y7zAyfan5q7CxOYnb8ec47ZtUXcTsZ9nC+BhAJmrq6u8kPbHprYNPEohAFeR+tDuH3RfYkrOfEhDIku1ZBXPr4SFeJtr8UEYIK/go2Ft3Mme48NunzYz1vMi7qaz+exGdc2fWEFYXGBlie2L670AOticBpcFApFsyGfimRU6UJrgptITQQGHCgkySQ1shBoHWqkvO4xts8gLvnMdmdv/RRjFtxrCH5Q+jWlMyhzdqJmZ1edMAfvC5XbAtkxs/w95tg+dMXLdp9kCNkyeeq9ndnutq77jJsR3D1X+h9ZBDrxIs5+/HpVodVYGIpDu+FmIFgW0H8/G2ZJSoF0s1K68fFzZ9UXzQM0xsC9cPO6Pc52d0UKOfv0+6Ujm40YXbEBmKd6nusTUEjxXMgbMCL7EFSCN+ERCEVGtsmDO2NFDHibVvp+3tfNm5UaObixrdk2cFtBSH9mGBa3Gih/yoxPn9fpA1wAy4VzJK4Vr2RSrJJgb9Zap08DoeA4D8ScifS+T4E+9bXZm23jVaPrOsU0kvWXuzvc4p6GG9TgA6wVCpFdcZ4+jZZ5Ivjs28aN5BgvccWnJ5knmf8HNAdKIKKWAAA="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514917.3/warc/CC-MAIN-20240719170235-20240719200235-00017.warc.gz", "offset": 109644433, "length": 10232, "data": "H4sIAAAAAAAAAO1963bayNLob7NW3qHD/hzZJwEkccfGe2GbJM72bdvOXNcsViM1oImQGF3seGblAb4nOZPn8NOctzhV3ZKQQGB8S8iEXGzorq6urq6uayN+bJ3tFZS8/CzzI7zKXVyPWIM4zB3ZlsuCxn3qQaMqq6WcXM0p9Qul2lDURrH+SwBwxjTb0XMH+w2y7TtWw/cNvdHtsZqsl1hOl6tyrsTK3Vy91qvlmFJTmNqt1OReb+dZZs+2PGZ5uUNm9b1Bg1TKarU6bhYE0dHINDTqGbZVGHjeaIsM3b4HXc0JUn+kjmZYPXuClpLW7WklVs1VK0BQSe5qua5ckXMKrVXKWkXTmKLtBChgZs13HD65nViQKteZrNdy5WpXzZXUejVXK/W0XK2o16uU6tWyWg+RHJzmWroOxLkNosj1vFos5ZVKJa9GjKZOn3m592cHDYIrchuFQtfomobtDdgfPssPDdeF5eqs51BLY/meUzDg3cf8aDD6t3lpNqnvDWyn4zL2wtCbRVWuBqhPHduzNdsExOpUk2e6sN3FcK3GaMCc3Llv4A5fHJ539t624J8qd05PDn9WinK5cw5vy5UQEb02barn9o0+c70GcQdUaZwet892i6WLytvz979Uiz8cvy1W1PLbtz9e/Hz6DpqDsbumrX1Ijjx8U3pdPf7PL+fv20etd2dnrf+e/7dd/PnoP3vvjk7KP++H3NRhN4yewfSIgGm5+DjwhubLj0PzWeZZ5u3FxSmsUwGplcmzjM4l+LVjvCJKnbzzTS7NJJDjGnlzdPEsowUi53HUHvvoFRDlFtEG1HGZ1/S9Xq72LHNJnesGaYHQjLxc29Js3bD6MJxqA5ZDJA4yemS7Xg5atA9N+RUZOSz2ZlbXswz7ODJAahrknHqviKyQd9TCNchElhv8n6AVyIGp7A8GUHo6GB1d76Ls5E5OW3v7u00ghVVw7+IStUUC5M0LnyHuHPAhB3woExWYEPJhixzRj7lWnzWLsvgzMdvb0/P2+fnBflOuVqvlosIqKu12e4pcZHJNKdIi7ZZlVS5WtsiIeoNmYWL8sHsyolru8OTNwXFTZybzmB6jbeAL2qgFmqYarVtJ0jZJVIAUKTtuHbWbQcNsKKC/ItdLlZJSq5bTwfZbF63d1nn7IUSaFLZ6CBKCwnurBLrMuWROg9iqe2V42iB3al8xZ9dGebmEM/tTbs+hVyac2FBWWSB+DdJ1UnRpqVQv4XHIbD/fP9mD49gmKNLk9P3u4cEeyeYKhR+Le4XC/sU++entxdEhgTNTKLSPsxkCf7KomUAxXV1d5a+KedvpFy7OxEFTFBwUvs7rnp7dyWxz5HAELbeZMlap1+tiRBaBGia1+s1sz8lGJ0ziR0xCTIzqO5m17SHzKFeQOZBh47KZjZ9SGCjeNbOzjmuWFCI0Fh2yZlbozdjQ06Nd8sax/RGHTQJ/YNdXYNrcGHigmInOiFDNr0j8mE1PqDNXc4wRKqkYmt1gzAvWd+gl24Kh5GiM+jVHnUKRY3dtL04PNc1gzue53EGbMI+4gINZPeYgJs0ewjk0uoZpeP8vl0tjqvRT7n0LDJ8ANJkUYZcO2s223mfSxJouDXY1sh0vRseVocNp19mlobEcf/OKGJbhGdTMuRo1WVN5RYb0ozH0h2EDyMza2hqg9gzPZDt71KOm3QdOMIuYRt9KYcl2QcBm1vDvtmlYH8BfMZuS612bzB0w5kmEuwYSlwnNdSUycFivKeULAqYALBmCxRC/8gDxb6WqypW6WqrIkuDlQ/D2gCU5esVce8geH7s9YlbOpZb7+Kgd7sd1dMMdmfT6EfGPht1SZ6j3YCLTtiyWpL1aexTcfTgJzDFYZzQAb+dJpghfPAlux7C8p0Ds0f6TMOPSwJNpMd9l6ehBHxHo6KDGRQdOOGoEdFB86vO3J2cXe+8vyMHeyXE4mzGk4CYWXPBLCz0KKgXOKfzgakhoU0Gm0Pu/00sqWrPEdbRm1rA00we9G+sp6A7td6yO7tij/O9udme7IDoegHJALd1kj4pyZI/8+cikCWTIE0LWjB7ZeK7bmj8EVuchtmibDF+6u9d74IW4x6C3N/8ihQK4n75DDIaa9xZ40iQI1fMtDXduw7KHHQ07N//C9jXwhAns0NBtWuyKtByHXm9sbkVdmu2DYZBFQ892yAYfAC3E2E6f+oL2ceIN6f9Im3mTuzEA/PLlJvmLcFrW+EIXGIx0bRibeW28mCYZL4CIFQA6pP9XTurLl7/BihfHvSUoQk/pk3glfjnMg6BRcIYvnrfjf+D+a1swk1Die2CTXQPMNN8SU+LeHAg+WDuXvD87dNEOjnc6M94KwkFZB4A2ILihYkOQvUD+HjhA5+AGxNaCPhHVPMbb/z2jvZFsBx+KU2/0NuJImyT7/uI1eFaBEASrFQRBPItuhG0BEkEYR/GJmS5LgoMHMGJxEMGisdRn7qieUs+8/btd0I3fDa8AKmgInZ5vgSIIfuU5AtBVtx4wwdnfbfCwe0YfOIuLGSGPTqxDiEchXnQgqoJGCHLBtWkQqefkeo6ETYa7z7o++Og9CmzAFlCZpyY1rHfgqY2HQtDxAdVeg/zKeYWuFiAChbtLXcZRcfQoAA0pny/wfxIy7jfs1NkIx0oQFXM1XThirosIzz3bYRKHAeLMLszTiA51Akjs6ZVh6fZVHlDsw5pxtZ/4HgUN+WEwBHrw4CcwbPzlOyZQV6C/U5GsgPDHB2cP378A4lm/GY6XXhEdAirNOwA/scG584lLAy4JqIVZx/Iwf4+4Rp0rBeEPUK4YYIR4HwMt8CXCvCjBOyhSNmx5xx3YVx3X1sBR7ljMg3jjAwEdubD+FzSmUegaQ9AwHZdRRxtMULcQ0vSFY+TQcTDYcb1JrMBYEbpltru2fk1sC9M1GB1wkdJZj/qmd+5Rz3ebEoZePJVy87dLRj5ERJq0lSXEgBHAVGRPdoHTGWnFHsxybIOnwQLNvWHoXKSR2b7ZtHzT3OJ6WKMu17IWh3YJtwoGLikT6tGhwJRuEnavD/QNiZnSS0N/Ke0NDFOXNgVqgTL0ojNcgT6P0G0uhlkgESi5Ag6HR4reN12OIOiYYa58U+BI2F+CBhjGBxaWCBMr9DNMBj2/Gr8hwpbnOUbX9wCRAetD5S/ZVt9kntsx3K7eCfbsZcDmiDSgTGARxn+t6zD6Qbz+FGl7ElkFHGQa8/ghZuWT8tlCy/TcNDbJwsNj5IYIwvGcZtPIg1KHUcdgy7YCox1YLB8l51NM2Pi5vRpQb+MP3zYg2NWFVyFyN7xRMAVkjRGp/dPpYefwZE8iDc6HmbTqxmWHfRyZHdD1EnmJePPcAOYDkQJCpS4mU6WtxTDNxGKBoZa2uL6dgyZgXhxTzKsiUkwSOoYF7oRxeTes8ZXOxBzizSTlaczcx2GsYMkD+brQ7tyFrYnFPxpLx5s1wdRATQcMRUluEvyZ9+xDTEnuAdPjnj4/PKnKdysT+uy+GTndUdxAuJ6YobpMI1BdKcqLj05TX/xMC5/+NhXWkV7immK6K4wGYORiO3GbkIxnmC8jCQ95Yg1jSp6PSceYXiIvXpD5YBA5LwLm+v3+AmBd6n4AD3zMrQXYFdMH0ahZLJuzbXmHAd80Fsq49Ar5K23OP8ChtRn//rQWqKVPSYn/hKr9UZwjj8K6XWA9ur93cQgXQPpo+IaGq035g8BQdLwk6uFgZGtT+jN3gEXHhiLLMkIDTAyUJ+MhjGFOdidqG0LD+K2EbzvoEjIHEAyKO2Hi++ZvTHnrjJg0LcsLkGK6BOLOwHbcTs+2PZwz2NsHMAJTU50w5xJjB+AMpzWwoNcJQpZgYVzcwUVVsyGRCVIlCw5en0I4hJGGC+Ot/k6LgGvhGSInzzyPwXsI9S9t38X4/5L9SRowvwDObOOWMyKS6hKwfx2o8xz4rwezi0m4PrIt4InhgNQB+YxZEtnZpiIwzuYnysb8XTaJA6tUgSMMs7gjau2cMc8Gr4c5hPpYohximuLmswa7Q+gl0wjE47ASxzJu/nbGfnQ+n4c1IILtAoX/no6hFtLNZZiFjMLUYI85Q3StuBD8K2IZ5gcj7kuAG3RFBzACAVJINhuOvGspjux/DiAG79MudQwvj3WgV5dEyRfzar5IVFmp4M0FuUrkaqNca5Sr5PcRzE4ti5H2xxH5Hz4pbhVIfTxnp7kuiFkexCT05ztiY0AHBg0Qv2PLVuZTJgno2FfTYDlonQbVmGmmwGIzaKcR1bHGl3OM/sBrkPLoI2LYFglZJDsUVY6SwYk0XX448LiJ7dy/+YzN/LhJAMZ8J9onAIod6RiOTux4I7Y4iMtYKETRe34eA8ENuvibTgIAIJwQDZxmw7uemIeD6OHhjs8agCd2IkFFsmcnsjjxQyvwmOySmWoMZE2wC963OHvgt8k+UksH+fzPyc9nbRKCBhyLj1z4zwQhCVnhJxP2GdmBdwtgD+OTjF+Gani6Z5FpQuwdLAzaLtMXnWbiTyYJEH8jzn3wyhFiEZx/xJ8AjaxOTO7Q1vGS41pMhhPriQN2PMNzYLf3A7cC9JKBl4KwkulynWVA2wgT0swjKcKfUDamRwGbNgiJSMwNnjVIRMs0s5GClcbWpBF1c+u/oWxuidxOPA3Toaa5EXRwR/3SNlwwbIlemNYY9kNtBy87I9N38f/YmokKiZgSh+X7Rk8iXdsBrdyUZImvSHRDrxQoZLHsxJrgjcbO+NLPORHZnRdW1x1tpUFrtmnSkctm8iAGsLE5vY6hbVgu/zG5knDgzLWEAA9dzX19BZeZTJtKeAkpTr7lAQ6KQKeP3rXteLxAEs+bYfQUQkcZhRC6ExTVN3Rm8gZMLsRqFkHQ5WDUNT/TioNfuH63GaJ+4dA/BUXSyw3kgd2LZtnkzr6Pd85AGesSFgqCLtIgkrT5UgJco5HTAWpi4zltU0OhNRqFnk6HWbAu33Gb4IasF/fH99fW1YqhQwveYRt77mskycJ8yDlY9yv5lSS9Ul4FqdMxx6BD3ow5/1MsnhqxEd4gjLN3ZlDXcyDG6QQZPnCL8B4EFoTAkXDw/kwzSTK+QBHbGNPEtw7Dt3npqhnTpMfJgeiGUxDxhwfNUcHN2OZV4KCiFg+X16J0P7uk5gaCYQiGVE9gJFEwNX4R8fWDYQpJ6XDaNyaFlXfhMpp3XPSYcZhEiNBsjsmP2mK5OxCWoX3JeF42NmgqLJw+yDFdAhHoOT/ztsN1nW3tQUj7oSlxIQJ7yhzY2OlTC/qbUNNrZi8cI2GMsoTfXUnrCOItDc4HuMB4fRDiDuaEliCpKrle7F536J9CUVK8K9OUurbn2UOJDLgObErFKRXKycRielKBgqQMxcUeaah1QAzgnAFWvrNCG8YjiKHtsI4wlW4UKIaeK4++Cfj1Axvmw5uOGAIZ1sgPVe7A0HUMUcR8wuz9+ptE+LTQ0ClJhZ05IzhfOnIHAcNB80fYownw9n/nD+gZzNTFmBhltKE2/nIbSrXx4g/f9rZc5hjA+T+ZHhhv0bzlNlQlAKENBYdUgrfn7dbZ3tsA7HjrUzRAUSKc3AtAosbY5GQnuBMjcKo5kW4A9UkqzGUzKuBoIQoHLuCuL2o8ZzsEtIca3+mgz8RgBIu5BXHJia2sg3kkvQkahkF3xL0hFrQBXRP+d4Qpk4IDI7XELJgriCaSdtJa43LNU2zIIIySYnZ6diUJ/4CjGmghd9p1+3Vub6Bgf2uWSvX6VmZKw2TiLm+WmQh2ypVWFMYI3DmhysKIKc7wcHKdjUyDCu0RaqbAQcVyMRanEL30iteXQU9N+6IbEoeY6Y1G/SSogfAS7VaW3zhBE5PmpQofKpsvgF4E7cQ337KH8LKJ3S9sfhPSYmZTyQa3CY1h+2M2xpKDYT9SlbqIXLOhKsvK2VDBZYt48THSj9lIP6KG69COUvmoVFBDZrk6lsIAQVzuZgTOkWMMXzCqQUywFcnaLWCFmXuSwzQXzRK8adTHncEO8VakFASYWKm4jgQNHVxxNnmoXKyPYkoGWJR0c4NES1PsDAmZLxJsHb4CaWfXpIbLyCnFa5YkRw6lgT0cMrwaako28y8dlnKUQyTCMQMvlxTIoe2Dpdo7eXPcvkgb8kuxXqtFnNO8j51L2CNozddqOVWWSy/ocLTl9Dy0Kp3e0GvipzPWiy27Z6yrr6EBXn9gl/Bz6H2En13b/hCOAQ/Bclgz0dQVU4kVrquyWCO8yMH/w3W1ylcKr5kHP6CjKtYbIXjweFSlbrNUVaF/FDXjJwyaSr1ceYGLhQ2OetBMwIBTcEVs1x4NjDGuETjmR4blG964CbPcTdAJhhu1UR8TdE2xC7FWnvJr8i2ahD1qnx+3zvangN8xak3CHrZevz9qLYb3zcnh/lHr+Hga2tcMNoX65GzvYLd1uBgZ7dft3R/O2lPAb0EKjElonpKZAo1SNtN0v3970D5bDPeb1vH+weFha28K/Ij6Dpw+2Hl9aorz05P22dnB1JiLwfpecb1V5wZtctDZ+/PzNLqQQRaTorOf+BVp365DQm00TqlMmBfuAk9aF000Ct8tO6RO37Bywn1sVEYft+LeHN7HSFgNdCCIZ39g4BuqvW5FL7OqXNJZrUbrtKIWFaYWq5TCjzr4pngwPAN9EKWqKiW5rpRrmMqemd7mqUbf5P5qyl0DofcC9ZN2F0Fch1zDK6YxHGmjU8p6Y9/lX6FWy55yCDSymjCy48K/JD5IATZ2lqUUg4VTYhrBDiUpQzJm0hXVz9IoOzjf3U+nC3vmUIXdSZoyIsxbE7k63xRJu3EaUXLsq3EBKJa3wzJnnLcT8QB3wbbCDcc9DhF6kbnCmkYQwvAUd8SIbj/Xd9h1MgkngTj90dEGcGSknQtM/fG6iTgfID4oQgn4gJN3t40cGy9eJKm+HkEM+zCywR3AXHwYCwv3ukEm1xF3QG4+zyJIpDTdB1Ekkt6YsZniZZo3P/0BRKVSUyWyk3QY6M4rIjCTLbIoomJNgVBtB1UgCezY/TBVKvWIImHk7oenCPoJEXETR0ITeD9ctVK0tsA43pMmuVoFRNx+kdB03hMVpt3IzkS5456bJ5dLEVmB0b0fJrVaRKoCi4uHJTLL95erMmC8GNx85raYBBb73kKhBjtpMRLY8TiqWaeV6cqDTmpbNzil00pvFuFcAboQHkesqKM8cycTDrzwQKN6bQrJ12D3H6ZdLOvmM99EoYz5FeZZSg+96FmUYA3gQZTsAQImUpZ3UHU4baRV6kVg3h4FR97BEjg5s6+pP7QtzkIyi3LX7z6Y+HPbd3NatIJZDJy5jICG8WHFExYLTPgCLH+YJ8qsZVg8/HnQKg6G+MlBvOWRsgUQV5HRTB4afz7MXXhtO0PqpUyrqqRMtOGsifGTqv4Dl30ocEyfW7zxcvN/wSsht461YVLwCM0YGhINn3lmsBL0sENDvZvPfRvj2DvLHJ89UuqlGkjc7sHJm7PWKdgGrnR4FLPY+BIePRh4eHJ+MjF+xuKDorr3MInlFBH95rNmDIH7d2ZCSMVYAaOVLNE3QmVwLO2fDs4v2scXB63Dg/OjNrimRyf77bPj4I1wWG9xTUegkmyeGH7Qeo/tIRk59siZudLAfRbOdIyaoMqfCQMFHqbGQ4VxsJNSONhayD0nvybd4t9IYWEfYOWkrpzUlZM6RiXOd178ygVVnvv4kaGiGDuT+KCLciVtAm7jxcstEljeJMTG3VyYNGcsQDjh3WwRZZPkA5Nxi7UNdF64snuZ6Vk4UtMhK0O9MtRf2FCHt5enDDivhGbGl+/QfvN7dh32kQ3BYuODcCAYHb9JXqkLboEy0wxurzYldXx9N4YimQPOiCTwYAw5MoPb46I+qXUxhtNZrksdPqk3mA9v4yW+Pfg5D9TTba1jGl0gF6DP/RFGKPMGhB/DHQ86xM/9Gi5vnTfSFbFbbDbREI7Z2Q+uheDzWnjObzAlQOySWVL8KnjEmlK5Vgr3PtnJ+cAFOdmbXPqhwfOfSZCUxYKHxoD//kfigj06oo41NWpqoUe0T13DSiXPNTyfzwEiBW6db3H5CfW0uBO/H92Xia7Jx6LtTFx0I5cz5YZlItU+3Z+a5E4W+hMYEl3RlfRYOeZLXTmoV0ulBa4cINgTXjlA9POvHHCIOVcOgv4veOUAZ/y+rhzgiu9+5YDvzIwrBzf/6+F9WaJLA8P1bMMJP9ozYpaLuT8X4xJ8iN8f/qKXD6ac/eW5f4Al3VqdLxmLwXhvIFg2rw3j1QGK9wNg8aL6i01xFkzcR3hEfMH9BCX1fkK1OOt+QvixLI6+Bhj5xEiNoQdTuomLC29A+YDwO/oidxfuWK6/aF2cTN8xOGOWoOShdXEU5Ketiyu1ilKhtFjXlJquF+WSprGKVtG1EusqavcJ6uLicN63Lp4Y/ah18RnqfMG6+Ey6HlgXn0HVk9TFxRqWvS5+LwW+qpDfoUL+mCmsUpWnH0Ej3nwmXFsKPKfOzecehZ1yXhHmafklLkKW6rEaZGRMlqkMWS0ucxmyinnCyQ9Tu6B7YInu3FLkk9fwlFUN759bw5uT2ltofBXHn+8dtI/3vtm0oMoLCKrciqUFgyWd58lFe+/t8cF/37fPMR2ICWh+Ms/RgOKzpl5YOnUHW2ED1rZe28a0TV2omJbqXQTFtDvZ9AeU1b6GZXtY5WJsfEJZSFggrF1AqJRWu1DuX7u4m/pe1SlWymj5lNGqYPDPKhjUa/V5BQMuVd95wSCRg7hXwSCBYYkKBtXyQgWDavlJCwZgCW8pGADE3IIB7/+iBYNq+XsrGFTL9ykYoJeTXjBoC9fUYlcenFGGXwrw7ZcF2mEGP76siVT/LTAifV8sF9PS95XaUqbvH56Pr5afNh+vqXJJVnus2yvWVVZktXqlrJbrellRtHq1XH+KfDzK/v3z8bHRj5yPT9WWC+fjZ9D14Hx8KlVPlI/HNSx7Pn6Oflxl3R81677KkT8gR16prXLk98mRg4Ff5cjvlCNvgRX6NjLkZf4hg9O3P59j3oVsBMmYzW82wYS+9I4q7y6SYLrm36KQJy3w0VLqxAvmtFMsdJDTTrGLT5q5fvI8M/j0KVNw/fCPzzOHZ3rJs8yrA33rgV7lhf9heeG6Wp6bF95d5YXjsfA988IxDEuTF67Wy8UF8sII9oR5YUQ/Py/MIebkhYP+L5gXxhm/r7wwrvjueWG+MzPywq8NdE/+8CkDIQI/xfIYDwP9IUxNmOvhB55c0+5jXIgqwSF96luGCSfaxT4Y4hojvNfACPXQ3xFQdGTDKLCrmu/yby12aDSSDRfMPbsUCCKt4/P24dFSZZ4529ZVOWQcJn7HrBNpYFUG9mFaOWIg3itH5sDvMRMFBEcQMhLecFbG4OPshLeCoTE8bDiR915qCkXWvV5NS7oX5bsn3fnp43RGnz/Fh1cnMvDv8tD/g2NYi2TgA5GbysFzgfzC6Xo8vk+brqdat6dUZKWo1nq9KlO0WrmowrRqsVqWqS4/frpeqKT7pusTox81XT/DiC2Yrp9J1wPT9TOoepJ0vVjDsqfrv7LZWpUE7vKouqJcwYA+YczJhiIXizlFkeubj/+IiguH6r625NWGcjVWbXiXJ2iblqnWUJSXudZQKaYm2oQnQJJ+wFctPdSrX6XyIH/9usMhCJh1v+oDH/oN1B5KaiV5I5YctfcP2j+0Dr/Z+63FKj5lRinR81i68qj15uD84kW779BLtnXWju6xnpztn7XPyVn78ODNQfv9TyRHzrme32WW/cLQDEfb8u5Zl0h1RYK6xFdyAB7yMKyvaQYfVlYZW6pQumLmCosqELOlTICaL6qpyPetqNxV0T+8wHJnvTWBSSivJS+yrDTXgzTXqgDzzyrA1ORKfU4BRgjVd16BSaQ37lWBSWBYmgpMpV6tLVCBQbAnrMAg+vkVGA4xpwIT9H/BCgzO+H1VYHDFd6/A8J2ZUYE5wC+8RjeNe6qUcC9WfHT01KSebeVJG0DAcqKji0pjn7kadbx/wg3++OLDp+4EDAifxBMwAV6N2YBP4/Gx6BCxYqL68QR4Rc1CVeupnxRQZxUtDnldAr/O3FjeTwSgfD5tiaGqdGuVKmNVpaZRuaTWa5UKK5bkcp3qNZ0+wScCxJm7b4khMfpRSwwztPSCJYaZdD2wxDCDqicpMYg1LHuJ4RH08qpMsPrkwLJk8yvqUmfzMRkBppIIQxn7Zgy5+tUy92Drv0bqXql9sx8auPXBOkuaOOKPlS/GnzrNE2OkdXzBryDnApVPNsLIQlyLB62/uTSPnuYUFiIjRM6YFVJ718x7qoUOMu8PsIvf+CcH1LQpuJIIs9zB0b1fnntaBW6h+lveB9OsDvOSHOZVMvqflYyuyrXqnGR0cfVU+UQYfK9UdALD0qSi69XqIl9kj2CPkYoGaDpsSnS92FIU+LFe3XXhl4ovVRVvCMPLXQPe4ozwEnuViuju+ZYQDs/uwKkxBTCHkAUE7fWCNY/7akHX7yBTPOOGHZyA0piAcAoE6TgMmDSJphRMkV9XX0f2YL34GizCenFfQHcCz2VdrcBSivvrqgL/cFUK/6dOLGg8mwiIxxDFyrzZxp7KrTNVJ2diIvaMTVWfN1XCb7ltNrU0OZvQ/b5l9CD4iHOzNG/S5KiF1lmenJlnQ2OrLM+bkAMvtEJ5cp7x1zXGJpu7utAJu3Wu4tSaxJcjTU1Xnbu28Tcq3crF2uSMoa8Tm6w2b7K4b3T3UzC0Pc2MbVpl7lxDG4QEjLlvenCqK0Mw79D6gV1f2Q5MVvFhVzt/+My5joiACUIqKh7twxz79od5UsRvMCwqRdF1h7tLK/h5hv1EymZqU7s+yJBnxCSopM6bL4RfeMYppaPZlsZG3qIiG4AvxMp6iiodcLd5URUXDbhtvvXqPmKrRgYFXeWxxZJDkpQIAAwVBJ5WGlApAgo8PNsyr1Pg1AjOsgNQAdVNn9KywTWz8AM38+gCt6IDxjLGR9FXGrcEMsrLKTFWBvRotC5Xunq51qvB/wqlvaJelDWlzipdCi+n7G9YbRnTNa66CN5KJK1S3aG/04/Bg+TqQbn6lTcw3HyfeS3Pc4wu2M4NiXsW0ubmFpn1kLn6LQ+Zq6eWskXx4WkeMldfvJT9D6hk44Lv9Yy5+qxK9r5PhjacZ+i1XWJK4DFc4vcE9gzL+PZL1fv+uirz9eGn22CFWFJeV6vBKuEdrnOiDn3HQYbbtSAGgHXk5Cr8U1R4XZl6Wl0ptQZdr8yqQV8w8wt+l8zZ2fRXyVDDdenMKnVwqBd4bl39aarUYxpE5BfkmrKTuaakqcdsEz9F4czJeXYguh5S07A+YCpqHMZ9/VjyC1+vKi52var4ONerFo9pccZVTLuKaVcx7SqmXcW0q5h2FdMuVUwrlyplqlRkrd4tlbvlGpXLrKTXqjqtyqUuqz5+TIsOwf1jWj563vXs4ozr2U8W0+KM31VMiwu+x+3s4uzb2Yc09hEzfsnhHdXsLtm12WD8fLZvN6o9pImnpkS3o/kq4bdY50RUe9dBImwtq+W0sFWt3+l5LzO/q3VMzv0f+3Lfa9YLBrAoaMsWwPIjs5wB7OzA8cvSUaosFMAi2JcNYHHGVQC7CmBXAewqgF0FsKsAdhXALlUAWy2XtGK5WiuVFaqr9ZqsML1eKxf1EoUftRiexwpg0SG4fwDLR88JYIP+LxjA4ozfVQCLC757AMs3ZrEAlnnEZHj5uGve/D1kwHPcAGoSZpEz33UBhPp4Gxskhug++engp5u/Id64+Vsz/5nhLvP4Z4PFQ067pgg28QzKIW8QCD9FLPiDjzr1eSTKv+cL+IRvsAV4JUbjx4gN8RK4dnvw/AVJCD7FDCrz4aE4C4rZhuXC0jgRPYdaHKZK+XNjOcyhINMyrL5D9UR0/haj8z28FY8fXvv6ETqepGWL0LlOWM4IfXZk/IXpqCvqIhE6gH3hCL2uTLryqwh9FaGvIvRVhL6K0FcR+ipC/7oRelmt1Cu03AUrrWkVrVquKLUqK5fkkszgb+0JInSY6gEROo6eF6GL/i8ZocOM31eEDgu+R4SOGzMjQr9wKH5GFGvLJiXOzWfuL/C3Ej6eFo46LnDB4HuXOr42wNHnpwfHJ7+0lin85ksV0WCieOuItsC9GkeWseVPRNGPgkkEwwposbTvIZn+8u8715ODLZiKV8Umcfq+dIALgrh0AS4eqSUNcGcGll+4FK4Wy4vcoQawL3yHWkVvchXgrgLcVYC7CnBXAe4qwF0FuEsU4PY0XdVqWrnO6nW9qvdKtXKV6TWVKWWN1kqxHXq0O9TgEDzgDjWOnneHWvR/yTvUMON3FeDigu9xhxo3ZkaAW86TMzjyDHYw5Rs7vvWS8nhps79/8g6QwV3pUnpMOv0RX4xJd23j8lEuOE+Cn75v772dAn8LvDByewPqmPw5lA+7Ew2Cs2wBKT8CyxmQzg4Etwv8Ecp/+LaHj3cMn3pl0csuf9LsgD+GS8NECD5o6xaCt5/ncpm1MJQD1eN6HXTzN/DHZmbtrwxMafQIf59Hoc3DsfIZ2SEKdEM/UEVN5ngb2WN/ePPZsfmjagEQbIM9IjefTXZ585k8z27y9Yzx9EDhuRuiNaHnoeHTrGm3F5y2S907Txm1oVGDpk+ZXA4fVRbwP/bw66zgNhikbUQuzImELyVCOScXec7lC6vrdPDJgpickl+YwQOjm1QiQwbAsKsj2/UksMMQNQwNsCUhgcltwn0MzZEUmSN+iPkjCtEYScIY8cZ0+JHDLifBsc2wwbjuZDhvlYKSNtQCuZocim3pE6HKmYTGNoQu4IKip5ZuFwI5jt5HT3vjr9bEmUj0ijPCNQHINj6Uguwf/ECGFNwLtDPANzg5DoG9jdSTaILtzETPTwzaOo7rot+RCfXdFADqCAgFR9w7mal80AZxlc+1LH8FOKrZnSNYIMiLS8ybz33xaE66Q8TzIqNJZ8x6xbqu4TGcGSYORDMA6PKbPB2UzGxA08DzRo1CoRvc8WFgkPJDw3UNfG4F3ujR4IQ4kQd0JLrwXL3mnUDsZFMKsWuzqB0Nu3hgAg4FxFxdXeX7tt03cWrigfZnXrPTNan1Yec1PnOS+BYjTmhRCb1kGnnDB8TnXosxgMceMFmw0ROrxwldo9/NW8yLlprP53Ea1zZ9L/giOZh5ZPsOpupBvegGDb6UDECzIZ2SIFTagdk4NZGYiNShEEOSSUpkIZA6lEiUQt4dSqKBT93NjvW6aBC7KbJ6+JTPlH3A53iGFjdgxEjJTj4fMezB0ob4SoUs+Jz+PnNs8AQwcfgsQ8i2aaSa0exOS9N8ZpjRVwVMQf8ri0+Id8D/68a/yo1LNQIDOMwb+pugWUD+/WzYJTAF3M0K7sb3z50UX1QPMBm7hB95zR5md/Z4Czm/+fvSEdNGhC44AZxTLW9oIxBI/rqQ12FHDoLLb2QPJME2DfH9dIR7X/eZpevnfc28XmiSw2vbmpwDv5KBc39iG2bPGgh/yolPP9fpGwzhlYtnJC4Vb0RTbBAnb1Jbpx8DLuB4DoR3E8p9FxxVRv2Omk08pjMxFL+2EQ6RUOYiV3+Lv2NYhtfBGAicgLVCIdIrzvPnkZ0nwsmy9WtB8cAb8m+/eJZ5lvn/ibEfz9gBAQA="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514450.42/warc/CC-MAIN-20240712161324-20240712191324-00031.warc.gz", "offset": 101668117, "length": 11074, "data": "H4sIAAAAAAAAAO1963LbONPmb7tq7gHR+yW0N9GB1Nm2/JYsKxNnfRrbmWNNqUASkjihSA0PdjxTuYDvNvbPTv7tNayvZu9iuwGSIiVKlo9xXtOT0YEE0I1Go9FPN0D91D7pFOVC6bvVn+BT/uxyzDaIw9yxbbksuLhLPbiolJRKvlTPy8qZ3NiQlY1y49egwAnTbEfP7+1ukC3fsTZ839A3qtUqbegyy1f0Wj1fYeV+vtGAJhp6pcpKiiI3y+Xt71Y7tuUxy8vvM2vgDTdIQykrlcllwRAdj01Do55hW8Wh5403ycgdeHCrNcXqT9TRDKtvT/HSoA0ViJbyaq0OvJT6jXxDVvr5ckVtVGXKmkyXt4MmgLLmOw4nbscbkZvNfqlR0kECSi1fUUq1vFqvKnlaa/Z1vVRugIDCRvaO821dB+bcDSKXmgWlXCnItVpBiQRNnQHz8h9O9jYI9sjdKBZVQzUN2xuyP31WGBmuC93VWd+hlsYKfadowLdPhfFw/G/z3GxR3xvaTs9l7JWht8qKHLR87NierdmmaBfGdvaGZ7pwvRx22BgPmZM/9Q0c5rP9017nXRv+KaXe8dH+L3K5VO2dwtdqLWyIXpo21fO7xoC53gZxh1TeqHR+Vd5Wfvjhx9Mf3v/y/U7noHP6o1L7+eyws1frfH9yHNTdMW3tY7Jm55f3P7bfVU/b78r1n05+7Oy0D36pH+6e7JyVD35oK+9DkeowJEbfYHrEwKxyfBp6I/P1p5H53ep3q+/Ozo6x/6C6JXL0P79bFYr81jHeEFkh732TKzUJ1LlOvj84m9Y8j33yitjoJtGG1HGZ1/I90OTvVn/Odxx6YYLkzmCE3D586FqarRvWYAPK+tZHpvPWLKYhbxvkI2PjPDWNc9DWH6lzuUHamsbGXlQPilNtyFAFPQcHamy7Xh6uaB9bpTdk7LDYl3m3vlvtfhoboHob5JR6b0hJJu+phTIokVJpg/8TPT0FBezY9kcD+nk8HB9c7qAC5o+O253dnZZGNVbDsY+r5SZhovHWmc/eEJjO730rD1KsEqW8UQ6luEkO6Kd8e8Ba5ZL4m6L27vi0e3q6t9uq6uV6v1RSdVUvN2oVrUZpv15vqDBhSyWlRDfJmHrDVnGq/kg9GlMtv3/0/d5hS2cm85ge423oY7/z0O+83KxH/ZaTvE0zFTSKnB22D7qt4ML8UsC/3KjJ1Vq91KinF9ttn7V32qfduzC5T2GoD0BDUPmv1d9T5pwzZ4PYintheNowf2xfMGfHRn05L8f1NtT0idqqTopBliv1Ok6n1a0Xu0eds1+OuwQnBDn+sLO/1yG5fLH4U7lTLO6e7ZKf350d7BOYc8Vi9zC3SuAvh2YIrNvFxUXholywnUHx7ERMVFnGSuHngu7pue3VLd44TGHLbaXUBSvcFDVyWGjDpNagles7uWh+SnyCStgSo/r26srWiHmUW8M86LBx3sppQR9xCYGK4lsrN2+y50gxasaiI9bKCeMbq3p8sEO+d2x/zMsmC39klxewPrqx4oF1Jzojwr6/IfFpNktQZ67mGGM0JLFmdoI6r9jAoedsE6qSg0nTb3nTKRw5tmp7cX6oaQY0X+Tze13CPOJCG8wCs4YtafYI5qGhGqbh/b98Pk2o0s/5D23QKVHQZFLUurTXbXX1AZOm+nRusIux7XgxPi4MHWa7zs4NjeX5lzfEsAzPoGbe1ajJWvIbMqKfjJE/Ci+AzqysrEDTnuGZbLtDPWraA5AEs4hpDKwUkWwVRdnVFfxvyzSsj+D0mC3J9S5N5g4Z8yTC/QuJ64TmuhIZOqzfkgpFUaYIIhnBiiPeClDi33IdfIKmUqmVJCHLu7TbB5Hk6QVz7RG7/9btMbPyLixb99+0w53Bnm64Y5Ne3mP745Fa6Y30PhAycVVN8l5v3EvbA5gJzDFYbzwEb+lBSIQfHqRtx7C8h2jYo4MHEca5gTPTYr7L0psHe0TgRg8tLjqAwtEjYIPipE/fHZ2cdT6ckb3O0WFIzRhRcDOLLvi1xT4FkwLzFF64GRLWVLAp7P4f9JyKqzniOlorZ1ia6YPdjd0p6g4d9Kye7tjjwh9ubnurKG7cockhtXST3WuTY3vsL25MmmoMZULIitEnay90W/NHIOoCAJSuyfCju3PZManrHoLdXv+bFIvgfvoOMRha3mvKkxbBUn3f4m7wmmWPehreXP8br6+cU2jIYyO3ZbEL0nYcerm2vhnd0mwfFoaSuNC3HbLGK8AVYmylkz6jAyS8Jv0Pab1gcjcGCr9+vU7+JpyXFd7RJSojX2vGekGbdKZFJh0gogfQHPL/G2f19evfocfLt70pOEJP6bP4JN4c5gHyFJLhnefX8X+Q/ltbCJNQ4nuwJrsGLNN8SEyJoTcHig+rnUs+nOy7uA5ORnp1MhSEF2U9KLSmw6opBgTFC+x3wAEChzbeF/SJqOYxfv3fc65vJK+DD8W5N/pr8UZbJPfh7C14VoESBL0VDAEoRjfCtqARwRhv4jMzXZYsDh7AmMWLCBFNtH71huYpdc7bf9hF3fjD8IpggkZw0/MtMATBW4E3ALbq2gkmJPuHDR523xiAZLEzY5TRkbUPeBbQpgOoCi4CSAbXZoNIfSffdyS8ZLi7TPXBR+9TEANeAZN5bFLDeu8iuAyrAuj4iGZvg/zGZYWuFjQEBneHuow3xZvnkFQqFIr8n4SC+x1v6myMdSVA1dxMFw+Y62KDp57tMImXAeZMFehsRJM6UUiM6YVh6fZFAZrYhT5jbz/zMQouFEZBFbiDEz/RwtrfvmMCd0X6BxURj5Gt++Ds4fdXwDwbtML60huiA6DSvD3wEze4dD5zbcAuAbdAdaIPi8eIW9SFWhC+gHFFgBG2ex/NglyilpdleBtVyoYh77lD+6Ln2ho4yj2LeYA3PhKwkUvbf8FjGoeuMQIL03MZdbThFHdLNZrecUQOPQfBjutNtwqCFdBtdUu19UtiWxjuQXTAVUpnfeqb3qlHPd9tSQi9eCjl6h+XjH1ARJq0mSPEgBogVBRPbonZGVnFPlA5tMHTYIHlXjN0rtIobN9sWb5pbnI7rFGXW1mLl3YJXxUM7NJqaEdHoqX0JWHnck9fk5gpvTb011JnaJi6tC6aFk2GXvQqN6AvoubWl2tZNCKa5AY4rB4Zet90eQPBjTnLlW+KNhLrL8EFGOoHKywRS6ywz0AM7vxm/I4Ntj3PMVTfg4YM6B8af8m2Bibz3J7hqnovGLPXgZgj1oAz0YpY/FdUh9GP4vPnyNqTaFXASqaxSB6CKifKqYUr0wvTWCdLV4+xGzYQ1uc8m0YBjDrUOoS1bDNYtIMVy0fN+RxTNj5vL4bUW/vTtw0Au7rwKkTshl8UQgFdY0Tq/ny839s/6khkg8thLq+6cd5jn8ZmD2y9RF5juwW+ABYClQJGJRWDsdLmci3NbcWChVra5PZ2QTOB8OItxbwqIsU0oWdY4E4Y5zdrNd7TuS2H7a4m9Wki3PsRrBDJHeW61OjcRKyJzt+bSCeDNSXUwEwHAkVNbhF8LXj2PoYkOyD0uKfPJ0+q8d1cDX1234yc7gg3EG4n5pgu0whMV4rx4rXTzBef08Knv86E9aTX2KeY7QrRANRcbiSuU5IJhcU6kvCQp/ow4eTFhHXE9BJ59YosLgbIeZlirj8YLFFMpe5H8MAn0lpCXDF7ENWaJ7IFw1ZwGMhNY6GOS29QvtL64gkcrjaT988rgVn6nNT4z2ja78U58ij02wXRo/t7E4dwiUbvrb2R4Woz/iAIFB0viXpYGcXakv7K72HmckMulUpYGsrEivJgPMAY5uS2o2sjuDD5KuHXHrqEzIEGhuXtMPB99Q+GvHVGTJoW5YWSglyi4d7Qdtxe37Y9pBmM7R0EgaGpXhhziYkD2gzJGpjQ6wWQJegYV3dwUZVcyGSCVcmCiTegAIcQabhQ3xpstwm4Fp4hYvLM8xh8B6h/bvsu4v9z9hfZAPqi8OoWDjkjIqgugfhfAneeA//rAXVBhNsj2wKZGA5oHbDPmCWR7S0qgHGuMJV75t9yyTbgsxc4wkDFHVNr+4R5Nng9zCHUxxTlCMMUV180GB1Cz5lGAI9DTxzLuPrHmfjRhUIB+oANbBUp/O/pCLWQb67DLBQUhgb7zBmha8WV4F+RyDA+GElfgrbBVvSgRWBACtlmo7F3KcUb+689wOADqlLH8AqYB3pzTuRCuaAUykQpyTXc/lCqk1J9o9rYqNbJH2OgTi2Lke6nMfkvThSHCrQ+HrPTXBfUrABqEvrzPTEwYAODC4Df8crm6ufVZEHHvpgtloers0U1ZpopZfEyWKcx1THHl3eMwdDbINXxJ2xhSwRkke1QVXmTDGak6fLJgdNNDOfu1Re8zKebBMWY70TjBIViUzrWRi82vbG1eBGXsVCJou98PgaKG9ziX3qJAlDCCZuB2Wx4l1N0eBE9nNxxqkHxxEgkuEje2Y5WnPikFe2Y7JyZSqzIihAXfG9z8aysvGeg67vd/aOTgy4JywXiildb+m+Ki4Si8GkJg4yywI0FMIBxIpOPoQ2evbMMmbD1HmYFbZfpy5KZ+ltNFoh/EZM++OQInQgmP7afKBotOTGlw4WO5xtXYgqc6E+8YM8zPAeGejfwKcAoGbitCNOYLjdYBlwbYzSaeSRF8xOWxvQotKYNQyYStMGtppbeNs1cZF2lyVKyEd3mS/+avL4pAjvxGEyPmuZacIN76ee24cKqlrgLZI3RIDR18LE3Nn0X/58sZSI9IkhitcLA6EtEtR0wyS2pJPEeidtwVwqsseh2ok/wRWMnvOunnInc9itLdcebaaU12zTp2GVzZRArsLY+24+RbVguf5nuSVhxbl/CAnftzW0dBZeZTJuJdgktTn7l6AZVoDdA19p2PJ4diQfNEDqFpaNwQli6F2TU13Rm8gsYWYglLALE5SDkWhxmxcqvXF9thU2/cuhfgiPp9RrKwO5HVNa5p+/jrjWwxLqEWYLgFtkgkrT+WoK2xmOnB9zE6nPeZqrC1agWujk9ZkG/fMdtgQ/ysrw72QH3UqkZOlwpK/LEa18hSQkWQsFBt9+U3kjSG/lNEDadCAxulNZjjv+MhGdqrIVbEOPSnQvo+g7gm14Q3QOXCPdAYDIInAgH9860kizjB9SwtQlPfOQQui0KVc0hk46RA80NSRDxxwFzlGwztngGOMimxaHyShTqZ+fUXMNiCL+Q66kWSQSkJh8iuX40TKEoPc772rSu8lvYjdYNOz0RHAYQombWJ+xH12JxO1CWkX3OeEw2VmkGEs7O45gpAfR5yqe87XBTZ1sdgLMfWxJXIlhOmQMDOztpwXwTanqt3JljJNaiHOH7VtJuBFhLg+kB7i9uHQTMwZxwIUhaSm4W1cse/UvYSYr7ZFqSanuePZLIkJvAllSesaCcTUykJ+0naMpIbOqRRloP1ADmGbTKR1YYwzh6GNkO64mV0o1AYui1cuRNwKcf2kAPdzki/DGssR9a3KGh6whPBD2x6v32u0Q4WbjQq0jF7QU1uFx6pR4WDCstrmGPp4p3f1hcoW8wUxd1YpzRDWXjb3dDrm+8+tO3vU2XOQZI/i+mB2u3uLzpbihyUIRuyFilFnw97bZPOu+CYoebn6MKshy1yZ0AZGrSWil5E7yJMTjUnEk3KPVZKi4UM9rfqCMyL1zEUV927ZzvD9A+Gnynhy4Tgxos5hXENSfWsx7GkPQWWBgGtyPpjTCZDc214P+eWMmkYMJIbUEF4wQRIWk77Wpcr3l4DQWECCm2TM/PIuEf+KmBFXJnPbffFt4NDOzvLblWqgebHhImZjXu8uaYycsdc7MVgRjRel4YsxAvxUUektfZ2DSosB+hbQo8VEwWY2qKty+94ellMFWz3uiaJIrMdUgnBUiQBOE52s0c33KC60yapyr8qFyhCMYRTBTXAMsewccW3n5l862QFjNbci7YTmiMup9ycbHsjQaRwdQFds2FBi1XyoVmLlfGrY+RlcxFVhLtXE/1zZ5c+yTX0FLmuFmWdnzTZJ5hie3hjMB8cozRK0Y1gAabkc5dU6w4d2TyGOqiOYK7jQY4PnhDfBVhBVFMdFZsSYILPd7pXHJ2uZgkxbgMiCnp7oYSF2G1HudZ2paLCskTuVkvizf8tg8rWBBhMaWrL+fUGhiAzsGbRX8IMLdHDmCk43QxfBv03u7jErRN1kJxYDsdOjSYA9jqv4PW1reKRsTfrJUIuRQuH/jPpEji0Dqtyq/lZqMRDYbmfeqdw8DD1UKjkVdKpcorOhpvOn0PF6xef+S18OTIy3Lb7hsvlbdwAT5/ZOfwOvI+wWtYvkB9DHa1Atqxyzx+1kLGpOnOiLdI5VWHhOM/wbLT85p7H9PTWhMXxbKZG1FnYFh5sXJv1MafNuMLKabBE7MVbTfx7I8MlmWlUmFqvUL1sqyoaqmq12W50exrfbVaapRoXSI6IADPQPOP2/IapXqzWscI4tyoIo/w+CZ3FVJSvMIQhOOTlgMW29BWcGtfrJHU6in5lMnC8a9w3HPHvATaN03Yt0nGVRI72MG6zTVRorZYEkwjGKUkb8jHfM6i1EUab3unO7vpnOGdRXzh/SRXq8LNXhGhEt8UMZNJFEdy7ItJ8D0WNsEUU0K+Uw4ZXwM3k2EMqW9beh4j0Roajd+s//t/iO2TcMr/Hmh7sOjHmAidc740IQzm82BL3QZcRnwragFsxFZR3SapngD6kMn9EXgCSi69chmCV1m6mwmcsU2PY1ywLI7Hdjz2HYiHv4qh2wpC4J4TsuJFnGNcP3DleZg30kh1kB847HJqEEHQf/a0ITAqbZ9hBIznDgRtmMs4nxPlA5W+h5UiToUH9pO9uRwDxrtbd2CZRC5CrCg0cYNM9y++MF99mceQiPi5d+JIBIQxoDEj43k6PnPCTyLbyUWPbr8h7SAcmc76JZj9u/FtWVdfuCzF8PONg/PEidowjxNrzHfr3YWXvREeF8HUXooYZaVMxvOIu4Cy7kT6LWAd6qWRbRJtNI8sHk7y79jpfdHG7NzEJOfV/6aGS66tawNRWInMWDMkqj6PeR7/uxPrHepdfRkAcebO05i5ys+ph7ov18ug+6e/HB4dn+398KF7GjOOc7gPciHe3RSOs0QAPGiACUx2416EXIQdqSoKdKQjI/9i2KLlpju12szp1xjK2hy6302rjHOw0/zM2Z/+3H4JYx0xEmRgVkMvgnuycT8i5gulhHUyJ+KpOBF3WLvJb8k183dAYveygolxLoi3vHhDDlOvo60XHzcJt8CFoH/XWMKETm2QW5nQeW2kemaZEc2MaKoRDbf8zBhXHkJcnSStcTrz/HSPfWIjMFR4elza7k6+JFPRwdYJZprBlg9A+pM9L7Emkgh+VUD44aTk2Ay2XImgnqaCLto6y6vU4US94eLyNia/O/C6qKgHHnrPNFRgF0qf+mP08BZVCM+uTCrt42EZw+VXF9V0xXMYYtTEhbDO9m6QT8FDzhwMDGd0h50zS4rvn4pEI9eUaiMc8+RdIQh5+may6yfs3GfTRVI6y8gxA/n7n4jrO2iRrZlaMx09oAPqGlYqd67h+ZwGqBSYdN/i+hNaU7GRbDdKNEV7y0jq1LJ1fb50ms9dOguVp1Z6luJZjdu9yJdM2daSDLHNFkiNbSXzK8kmEveibYCxYOyjpXrq9XpjmVQPlnvIVA+2f02qhxdZlOoJCjxmqgdJ3kuqh84kesINYdckeq4pdv+JHuzy8oke0UgrGJw5gGVfao9xIb0cuxwDoJteICcABK6+mOLMvThgzKCTo7EbbLo+N2zQY43hE0N0H50813DSsM5TSOCotv0xytYMmOWwVuKSKkjtv1TqE2G8VEo6vnCBwPvJy075Zbt5Lt6EaOAyCgfeTCzKBRTWMym8hGKCj8zDOz6+iCZAYBEHX50BQPBWq1kH+YKEK/lSo9Gs5OsT/ngQr6VUq0gxuoyJopZSAvQcANDoDu5xgAr7fFVBksY5oKjo9thXWx3m9CcX8IBGC6yc4d4k3TZT9uiw+337cG+m8J5L+bo2XaHT3t8/+rB7P43fOfmHE/Vhk3/9Ron1S3WA/qWqXparFapXlEa9zqq6UtKU2i2Sf7g7hMMozx5L12UChSm6dSYwUf1+M4FzVrBlM4FzObtrJnAOXw+TCRS9mJcJfCopqPtcs0L0vkfw6X10jMy4RCaynGWr7j1bRTaXjRlW5RLYou3QspLQ9t6uNaVeC7kKDL5o59i5+tKnmsGcN6AUWuG++Jtud96QMV2+03B1dYMLYnbyzOsFn0jukE1EA73gyy6MOa7I0dGrr55URL9ibo7K5lv57xBdhQYE4r2JriPZSANAcMK9IcK54WFJyx8ViFKupkcCHiEdCi7a10iHKjKpktX5GVF8vs/dOn26c1jcOz09LHbbh/NUZsaB/boJ2m8hw1pDPe4cnRx/OCXvj961Dw8xRxBLESzVjFJCO7Lf7Zx9OOmC5Sf7bbKzt7N/44ZqmKs46J69O9oV7Ygmv9WURRN601ZqsYxFwmTwyM+Uk7FcsjXVUQuSrY/jHj1YQvC5OglpCc9brOPhiIWL+RvCAXpK43ypCLOpoQFPFlm7/VK4icvgOilEWXq04JiXDxmcNdZ3SuU++TRsZmmfpqWNZ2SnohnJ3iWOpJ7jE4Y9j8Gl6fOo54bOCmNrEKH8/Vf/KpWbm6lWOYwWn4cfbmChwypgp3NTAxEe5cKzS8FH0Bse5ZNg4ikw7fisq0t4GrTHOccuDDD2A5+5nCd38ADqy3L7pfIW/olu5plfoCP6Fz4E2R5Fl+HDMS8mwz+kJz4VSo3Cwa/ir/DHePBKBLbQLWy9eDH58uLFK5AsEG4B4+JDLgjKhxKXkrqZ5c+/7fx5Qy4vyhDzOb0oB8o3Y/yH50ATwcXb5UATTTypHGhzyRxo84FzoM3rc6DN63KgzcfPgTafXw60ebscaDPLgWY50BvkQJtyvZSXZ3KgteeUA72PlGbzYVOatEL7NcbkMoBpSisNpaTKlbreb+iqputq4xFSms27pTSbD5fSTFuQbpDSTOfsHlKaaXw9WEqz+UxTmlNBOzBrSpbUfHZJzSz5+MySj7Wvl3ysZcnHieeYJR+zkPi3GRK/Pvk461I9ZPIxxZF5tunHx08T1h46TVhbMk0YmNUsTZjZxCxN+GhpQpx1cpYmzNKEXzlNWFl0UjJLE4ocX/PuacLm00sTKo3qMg/FxGIPmCTE5hfnCHmJBSnC4P4jZgiR4vNKEGKPb54f5CMzJz24y38MIHjChEveAzbx3W88z7eLS3ApyKYFPcM8W5AV892ptNwNyos0mazUZ9Nk+MiZeWmy98y3mDtJ+fHUXd82EvmytyZgqhixW6bM7prVQm152KSW2qC0Uq6US7Sk64qqan0mU11rVMqsWavWK/f/kE4xA26bxkrUvtcs1hybuWQSay5fd8xhzeHqQVJYog9PPYO10Epm6aYHf+LjE0jnyJVyLKET2Oqnk9PB1ecJ53QqlRIfdrEMBlgeFsBJaqf51RI7sJp/lYdsNrKHbD5CxBHVrnv64fTGsUFU2MOjDz922x/IWff0rH3QPTz7VkOD+PCD7dpPsdBgt3Oyx+Onp+29w7MuWeOLiLtO8gR/dN5IhOBcog2dENZ4BrNSFr+l0iupy32QXUldZB/10Yy3Wg2iKPxkSXjDnz+Z/pDHeuwhj2gBbpNsWGRNN0lzknG4ZR7hWTw5MrMMj2wZsqdD/meFrZVmSVkQtUadeuZB6wRAv1XMOtHCkwlZlxv1JSLWUOoBA9bQ+uJ4NRZYEK4Wtx8xWg0En1ewGjp881g1Dsu8kyzo5lgD/G1w4fG8tQ2iU8vFZ3v/N/6eLT7qG/PC/BojIxvWOvzFPp+ZS4a025233Z2nFMveF+Fi3u1E9Bg6j9+hq3jlpVLHaHWjGYghOg4S3McLXBzwLgQyfTTl4ekER0yU2s1i5/dxxIQP653OnXRPjj6cnoLH9/NM+X3bn6XYaf/a3d/vns6UfgdDbdw1Rg+z5GFD9HpFrdaUcr/W1Bt9taKVVbnSUGE10rQmY3X9/kP0fOLfNkIfr3yvAfr0VWLJ+Pw8ru4Ynk/n6UGi87wHTz04f6d1IcRP3U8AK6/+cXlBDUGmNsSvWZD/FkF+pQmgMlwrcVOaWFVvs3P0vvazyjVsh5tqEjPmt2QKZsU2N+QktPPZ8ZTnkspYcDylrny90ylK7aucTlFINUtjPHywsoLWq9M+63bedU+zfc4Pc/YjzdsJT348pI9xhxxHttI+9MmReZkcbnCjkyPCDN7jwZG6kmVxMsP4VQ6AZLmZbz83Uw5/szY7UDAnNxMPztwqNRNv4MlkZuRmbamfXYJi95GbgdJ01JLoy3JblvFwVH3HhTeFn5NSDB1eXpZ3DLwNFOEj3pVr4nbft4RqeDYsKaYpCvMSJVGC9vtBnyf3GsGtP0CjGAad8QZnoDJhICSBRXoOAyFNN1MJSBTw1FZoVF+W34JZfVneTf6I6kulBl0p7wYHunTxHmN3hpqIwkxKlGuLqE1cnmsp1acpMREJiJFqLiKV8Hyuo6ZUpqkJy+9bRh8AYVyalUVEk7WW6md1mjJPBsR6WV1EkBdeqoelaTpaFB+IEVvYu9CZu5ZWeaZPvppKrr6wb6LSUlJsTFMMHYYYscYiYnEH4+azYGR7mhkbtNpCWiMblASWct/0YFbXRrC4w9WP7PLCdoBYzYdR7f3pM+cyYgIIhFzUPDoAGrv2x0VaxJ2/ZbUo8hRvrq1jUED7gYzNzKCGPw0do6csoheWX5rijNHRbEtjY29ZlQ2KLyXKZoopHfKftl3WxEUVrqP3sr6LrdWjBUWjjjdZsUohS3JUABYqQOhWWqFKVCjw72zLvEwpp0TlLDsoKkqp6SQtGxwzyx8t5gvcih4sljE5inuVyZVAR3l+LybKgB+ZVWuyrjW1alXt0361VK2UG02qq1qlJlfZ7Pobpv9iK3yUBhSylUja1o0e/YN+4k8lBYcg2MDxxhsabmHAvLbnOYYKa+eaxD0LaX19k6Q/sBRrL3peqbg/u71DpMQe4nGlQHH5/R3/Ads7sMO3eFYpDsz8DR4jwzJc7+ofEUzDeO3kihv8bjvgyUPbP2fUJ2cMSGKnv/ETi3z3hegpP3TYmOysEGcRk/fwCpdFuOUikAd8iiSStr/joSiInR3Vipy6s6Myb2cHIBDbjHab7BrMT2zuOGW+Yd71NOR02Z2jw19mCh7DzZkNHEdn77qH3bOZwl3d9qmjzzJxuDdnt8fsQ0xPDve6J7NsGAim5+4NCUzXtU8lhRn2MJtDJjwIfBuEpXLTYamkQ8PDdGgrQspJOtvHzBlR07A+YthqAla/PmJ+XD4qyjLAHUo9Lm4Hghlsz2B7Btsz2J7B9gy2Z7D9ScF2vaaV6rJcLWvVcl1tNuu03q/W1YpOGzVaqsVavi/YDv7A7VE7Vl4A2sXtR8TsQPBZQXbo780RO47KXMDucldStc3o7O7VF9P4c7Ir4Rs8cIHYN+pV4sFAwS9z8P7N4OsbVpv+QQ4FZu/sD3JUqkoapm40n9RpibtCVlCxp4ZYcao8TcA6Fyg+8hnA+nKHAOv3cwpwecSKFDPImkHWDLJmkDWDrBlkzSDrk4KstWa9pjbUBqtrfao1+kq5We5rDY02dCZXgOt7h6zoENwes/Lai54jUJ/zIIEHQ61I8VnBVuzwLZ4kUF/wKAGKDiUANTzNMXCu/pfGvvUMMhXwM5ZQHTgceCoam8aqS5WNAdRmSW6U5LzSyJenAWoZHJs0gCrPAFREoCdgZxlMGxb9iKWrGcHPVzrMNAYGg8X3vnO+3bP23uyx/RN6iYd0ZhPEH87O0rKzBwbwPZMm3mmffOj8MFO4belCos0Zxj8c77c7szno1KcOfDg+OpzNQL+nWiKUEJTeB2tZOj46O+ue7M0KZm9gUU2M92NCe5yFTw3bc3vyNMH9fFT9yFnxWqNUW2YjOZZ75J3kSDID+BnAzwB+BvAzgJ8B/AzgPymA31ebSr/epA0AEzWqVqrlPq3o5bIiN6rNPq3fP8DnHsEd9pLz6os2kwcFHnM3OZJ8ViCf9/gW+8n52MyB+WKDMe4kd/CMtZdMUX/jiD/aPc03cQdoEzo5L/U8FQS4dfVYXKDRrCilfK3RzDdm4wL8MX+FZGAAxFF7yMz1TaICZ+8AJ7c/7Kfgayid/+WcP7fmbluxUTmfGvoVE+1pwt8FsPNxGanVq80l4C8We1z0ixQz8JuB3wz8ZuA3A78Z+M3A75MCv01VVuuVUr9ZaTb6Wl8uKapeqVVrFa1fU8u0dP/gFx2C22NfXnsB9A3uPyLyRYrPCvhih2+Oe/nAzIO9+NMsvsWIN7z6Ypv2wAifWgj2ynOWBb7iIXCgD+SgfXK2d/ikwK+NvloJOgmv3lDgVNHVxNPrRYenoe/tKk/v2K6W63K+NnMKWk59vn1zLu7tAF2PCujbN3Rm3Ar5iiGagbI4hrx/dfuefkV2SeiL+vnUkC+faU8T+M7Hm4+cf65Vl9nUXas+8p7uWjUDvRnozUBvBnoz0JuB3gz0PinQq1T6tFJtqnpT1foU0GtfrTeqTUVtVvp9gLP3D3rBH7jDjm6ovGhDN7/9mPu5a9VnBXihv7fYzQ2jMn8z9wn+VrDjBD/sofukMwSM5PFniEkseuw+Lgv8EcVL4V98KhTi5tMPJ9//8pTAL9+/fSJg66TbCF3x+Vyi69HDvV4qdfZJlBXp3IYrfqSNiyJtZ/h9tyxwsdyU03Bx7UFPMvOhS33cV+pG6O9PABHP5oPFc7lmf+Ht5KT76+w2ceo7hjbbdPfo5PtZqN32B74Ly+mjbsmuVZ8aMkeT8DSB+VxA/Nj7wstLAXMo9tinrcsZNM+geQbNM2ieQfMMmmfQ/GlB87peAmSuq029Uan1S+VSTa7ptUal0mhU1QZ9gAeEoUNwl9PW5cXgXNx/1NPW5WcGz6HDtzltXZ4L0Ns+AWwGJj7IQoujx9/49us2ImLRrUTSWHRuClcvWTiWYS7BP7lclRv5yjSSVirl1Azz7JHrR3zO9t3PCZefHirFmfBEYelcPPjIG7YrSqO8zDlhLPfI54SRZAZNM2iaQdMMmmbQNIOmGTR9UtC00Ww01XJFkas6gNSaLNf6DFFqra419Ypeu39oyj2CO5wT5tUXnRMOCjzmOWEk+azgKe/xLc4J87GZA1DlAtmHS76DT58i1LLQegS/NnUAY/uNQ9X9l0o97B0AwKB/sV94wj5OZ4JvVGd6a3Sj2ijnSzNbo+uVWeAK/X7QFPA9HwmernLcTv0pp33bN+5+fBh19qmBYjH/niYqXoBGH5mRel1WloHFWO6RYTGSzGBxBoszWJzB4gwWZ7A4g8VPChbXqNpoytVKs6apFa2qlJtMrrAmoF9drvS1B9hMzT2CO8BiXn0RLA4KPCYsRpLPCxZjj28Bi/nYzIHFSgaL7xkW46OymtOwuCan5HOh3415sNi1fRc9j6eAh++McVEBnxzG5ZPpiWLc+dByq6iatvbxT9/22PZqKGTJoucqdWCghjgWWxpYJebA/Ws43nqRz6+uhNAQzJjr9RA2rOHL+urK36tA0ugT/r2AqlyA+ekzsk1kuA33gStqMsdbyx36o6svjo37P7AgLDX2mFx9Mdn51RfyIrfO+zNppw/G010TVxOLBlz4PI/s1pJkVeremGR0DddIuPR5NZ8HCYXy56IOtZdLG1a3LWxcrE0SfpQI5ZJsSdMKOEGdfNeBIr+yVAdA9cDCcxOVVya6FeB8tKhERgzKwqCObdeTYFkHEDIyYFkK+UuOEg5juLJJ0crGp3B+4LBLXNcksa7xi+nlxw47ny6O1wywQ1CDi1YuymlVLVCr6ap4LZ0QGp3p0ngNSxexQ/jOrchWMVDj6HtoXMSnFTElEnfFFOGmAFSb9A1YlfZ+JCMKvgouViA3mDgOgaGNDJS4BKO5KpasybWe47row6yGFm+mANoIQJZj7unMtT64lvF1gNtY/gnaqOe2D6CDoC4uMa++DEChXbQ3BFTDHW9GROdQvWCqa3gMKQPhQDODAqqhmobdQ8XMBTwNPW+8USyKO96QAbYrjAzXBfI66zsUlrhC34mcqQNxC6fVW34TmJ2+lMLsyjxuxyMV50sgoYCZi4uLwsC2ByaSJh6Yf+a1eqpJrY/bb6nhMP6EEyf8QQlCz5lGvucV4rRXYgLgUAaIBQM91Xsk6BoDtWAxL+pqoVBAMq5t+twKmoYKlMf4gBWTErAuukG94dU/KLFCIRfyKQlGpW2gxrmJ1EQEI4UaktWkRhYDrUONRC3kt0NNBLDj2LmJWRcXxGiKKOHWsJw2DnB1NVxzA0GM5dx2bMbE76AP6ZucALiv/i5zbPAFMBD53SohW6aRuo7mttua5jPDxM5uFaHUTOl/QRnfc5hLQjXjUuNajYWhONANPVewLKD/fi68JVoKpJsT0o2PnzutvmgegBg7h5eCZo9y2x1+hZxe/YPXEowuSQDmqVYwtDEoJP9cLOgwInuWC8rie6QDmmDzZ6Wi+LnzdRsqql/wNfNyKSL7l7Y1TaPom0L6U8Mwn2qg/CkzPn1epw8wQDUX50hcK74Xl2KVOHvT1jp9GnAFx3kgnJtQ71XAr4z6PSXU4ZSqAN0A7lnCmIvo/zXujmEZXg/hFPgAK8ViZFecFy+iZZ4IH8vWLwXHQ29k4qfvVr9b/f8SsWIFdEQBAA=="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514450.42/warc/CC-MAIN-20240712161324-20240712191324-00007.warc.gz", "offset": 6207916, "length": 7512, "data": "H4sIAAAAAAAAAO2d+3LaSJfA/w5VeYcOXyWyNwEkAQJhw5RvmXHWTry2M9eaolpSCzoWEqOLHc9UHmDfYv/byXP4afYt9nS3JG7CgAF/Uw6ZMRZS9+nTp0+fPr+WMD/tnR+UlKL8PPcTHBUub/ukgXwS9D03IPHJQxzCSVVWKwW5VlDUS6XeULWGXPs1LnBOTM+3CseHDbQb+W4jiqjV0Cq2YhqaVjBNxShUNEMp1C29UrC1sqlV9IqGddJ6njvw3JC4YeGEuJ2w20CVWr2qDU4LhXC/71ATh9RzS90w7O+gXtAJ4VJzTNWfsG9S1/bGdKnX7RqxK0ahVq7ZhYqC5UK9rFQKRq1s4LomE02vt2IR0LIZ+T5v3BsWUiXENBSNFBS9XitUTFMvYFw1C9jU1bpMlJqi6omQ47PCnmWBckEDKbJeVMuVoqJpRTU1NPY7JCx8PD9uINajRqlkUMOhXtglf0Sk2KNBAL21iO1j1yRF2y9RePe52O/2v3OunaYROQ4Jqdu2aNB38O0rajXLqhKLP/O90DM9RwiHAU4v4FvHw1bhkHZIEDZQ0MVK43Lv/a/7P7378eL0/d6vP17uvz344fzjzz//9Ms77UA7/6+DuO6+45lXozXPKgdvP559PHh7dPHu7HKv/PHyp8rxx8rJL7XT9+rp28RBji0wJ7UpsVIFJgf2czfsOa8/95znuee5Hy4vz5ja4HYy+vCfz3PCCd/69A1SVPQucrhDotgVNfT96eW414Tkc1hiQneQ2cV+QMJmFNqF+vPcz4UDH984xC9cgnkDGw6OXNOzqNtpQNnIvSIWl+YSk+nWQFeE9AvYodfgaT9i/7aB9kyT9MO0HhTHZpcw9wl9Zvi+F4QFOGNeNeU3qO+ToTfTLj3PHX3uU3CbBrrA4RskK+gddpkNZCTLDf6/6OkFOM+B511R6OdZt396u8+8p/DhbO/gcL9pYpNoZbnaHvapHUSE8OZlRN4gmM3vIrcAVqwitdyoJFbcQaf4c2GvQ5plWfwba+2Hs4uji4vjw6ZWNbRqrWaVsW7UqxWjLss1WdEMuyJXrIpS3UF9HHabpbH6PeNDH5uFkw/fH79vWgS8mFhDunUj1u8C9BsmWi3ttzKq27hSsVCm2fu906NmfGJ6Kaa/VpEVXdYnehgXO9y73NvfuzhaRskTDEN9Ch7CnH+m/14Q/5r4DeSpwQ0NzW7hzLsh/r7H/OW6POy3iacP3Nbws4KpLOtsOuV2Xxx+OLj85ewIsQmBzj7unxwfoHyhVPqpfFAqHV4eop9/uDw9QTDnSqWj9/kcgn/5ODTd3NwUb8pFz++ULs/FRFUUVik5LlqhlW/ldrlwmMJu0Myoq+i6LmrkWaGGg91OM2/7+XR+SnyCSkwSwVYr92y3R0LMg1gBfJheN/Nm3EcW/qGieNfMT5vseVRKxbi4R5p5HIVdzx+qena6j773vajPy44WviK3N7C2BUPF49CMLIJEcH6DhqfZZIMWCUyf9lkgGRKzH9d5RTo+viY7UBWdDkS/5aIzNPI9wwuH9cGOE7f5olA4PkIkRAHIIC6ENSbJ9HowD6lBHRr+X6GQZVTp58LHPfApUdAhUipdOj5qHlkdIo316ZqSm77nh0N63FALZrtFrqlJCvzNG0RdGlLsFAITO6SpvEE9/Jn2ol5yAnzm2bNnIDqkoUNaBzjEjtcBSxAXObTjZphktyTK5p6x/3Yd6l5BwuI0pSC8dUjQJSSUEM8NJO4TZhBIqOsTuykVS6JMCUzSgxVH/CpCie+UmiprulrRZEnYchm5NpikgG9I4PXI6qV7feIWAli2Vi/a54lcklKsUH6/Z1TaPcuGhhy2qo7qXquvRHYHZgLxKWn3u5D9rKWJ5GAtsn3qhusQHOLOWoxxTdnMdEkUkGzxEI8QXGiziMsSQJHoIYhBw01f/PDh/PLg4yU6PvjwPmmN9jCkmaWAhqRkYwgpME/hhYchEU2FmiLuf8LXWJzNo8A3m3nqmk4EcXfoSsnycacN+bLv9YufgnxrtyQuLCGyi13LISsV2ff60f3CpDFhzCYIPaM22npheWbUA1MXAS6OHMIOg/3bAwcHwXuI29t/oVIJ0s/IR5SwyDujPGoiVsqOXJ4Gb7ler22yi9t/sfPPrjEICkkvaLrkBu35Pr7d2t5JL5leBAuDLE7Yno+2eAU4g+hudtOXuMMa3pL+Q9ouOjyNgcKvX2+jvxDX5Rnv6ByVmV5bdLtoDjrTRIMOINEDEMf0/42r+vr179Dj+WXvCI1YpvRFHIlfPgmBGoVleOf5efYD1n/rCWMijKIQ1uSAwjLNh8SRCMvmwPFhtQvQx/OTgK2Dg5HODYYC8aKkDYW2LFg1xYAw84L6B5AAQUI73BeWE2EzJPz8d1PON0bPQw7Ftaf21rDQJsp/vHwLmVXsBHFvhUIAtCyN8FwQIhTjIr4QJyCjxSED6JPhIsJEA6/PLRieMue898krWfQTDUsQgnpwMYxcCATxryIXALFq5gQTlv3kQYZt0w5YlnWmz2z0wT0BngXa9IGq4CRAMqQ2DSTZfsH2JXaKBofEiCBHtzGYgZ2BkHnmYOq+CxhcJlUBOq5Y2Gug37itWKoFgiDg7uOAcFFcPEdSqVgs8f8lZrjf2UWL9FldCaiah+nSKQkCJvAi9Hwi8TKgnGNAO410Uo8UEmN6Q13LuymCiEPoM+vtFz5G8YliL64CV9jEH5Gw9VfkO6BdCX/CYrui51kRJHvs/StQnnSaSX3pDbIAqMzwGPLEBrfOF+4NrEugLbQ68If7x4hH1Hu9IHmB4MoAI5G7CrFgl1TyvAq3mEt5MOTtoOvdtAPPhES57ZIQeOMKQYycO/4LHbM0DGgPIkw7INg3u2PazSU0u+OMHNo+g50gHJcKhhXolts1POsWeS7b7mF0wF3KIjaOnPAixGEUNCWGXnwr5e7vAPUjICJT2skjRKEGGJWZJz/H7Eyjog2tvPcg0yBx5N6iFndpZuzIabqR4+zwOGzigEdZl5cOEF8VKOtSLomjPSEpe0nYvz22tiTiSK+p9Vo66FLHkraFaCEyyaJzPIC+SMVtzydZCBEieQBOqqeBPnICLiC+MGW5ihwhY2T9RWwBhvrxCovEEiviMzQGV36jvzOBe2HoUyMKQRCF/rHgL3luxyFh0KaBYbXjMXsdmzlVDTQTUsTi/8zwCb4Sx1/SaI/SVYFVcuh99hCt8kZ5a8nK9MKh22ju6kPqJgKS+lxnhxYhqEOt97CW7cSLdrxiRcxzvgw5G5+3N10cbv0ReRRg1xJZhdi74SeFUcDXCJKOfj47aZ98OJBQg9thqq4WvW6Tz32nDbFeQq+Z3CJfAIuxS4GiksE2Y6Wd+SRNleLCQi3t8Hh7j5jYeMOShrIqJA15Qpu6kE7Q68WkDvd0quREbm7UnwbGXY1hhUmWtOtco7OIWUc6vzKTDgZrzKhxmI4Nyjy5idhrMfRO2JbkARh9ONPnkycz+O7kkpw9ctKkO+UGxOPElNDl0Dh0ZQQvXjsrfPE5LXL6WSGsLb1mfRqKXQkNQM35RmKWkwxauN9HRjLksT4MNHkxUJ0xvYRevUL3FwNynqdYEHU6cxQzcHAFGfjAWnOYaygepLWmmeyeYSv6BOxmksTHpTfMvtL2/RM4WW0Gv788i8PSl1GP/8JC+0qSoxBDvwMwPUt/F0kI5xC6Mnk9GpgT+SAYlCVeEg5ZZWbWpvRn4Zjddmwosiyz0lBmqCjfjAeMIX6+lZ7rwYnBW4m9bbOUkPggoFtuJRvfd3+zLW+LIAdn7fJCSdHciOB21/ODtu15IWszHtslDMG2ptrJnsuQOUBm0ixlN/TaMbLEHePuDimqmk+UHFFVcmHidTDgECONAOq7ndYegtQipGJPnoQhgfeA+tdeFDD+vyZ/oga0LwrndtmQEyQ21SUw/0vQLvThx4pbF43weOS5YBPqg9eB+oS4EmrtYgHG+eLYjWP+Lj8qA47DOBGGVoI+dlvnJPQg6yE+whG7Rdlj2xR3X00YHYSviYmAx6Envkvv/vYHeXSxWIQ+MAG7JQw/ocVQi+nNfZgkhmJbgzbxeyy14k7wr9RkbH8wtb4EsiFWtEEiKCAlapNeP7yVhq0ejwj3kRsfCJj75HQjct+Jr8dCTez7MBgBcdrJ3fV0HkjC15IRiM9ywY3Kyx0ptbc0z216NpVor4Mwu8cBU4RaFrvpkvip2PsEo19T0Ecp9t2OVBoYdEIBpVxUKmq9WhOvTJ1XrhH0dx5aPDZIHLln1l607xWlrqAC+hH7FO8aPiq1PmH3mjmYKivlkX4mQyOeyODj0h6M04pVU7hq6qhq9t3Xa/8+3dZiJZWrUh5V5RT7weOqUeZqVEbV2AN7OI+rR4XrUR3VA+b6pBZLzkx1npnpwgIzMStFfGChLicORLhjyxgTOCqB3c9gztxWtM+KVuxQm21DotwUXUd3FLim4Kt3/x2y5YxpkCuiXYM7727JaPHFMzfsOOzui1heLRwSMdsbCImDEQ/PiRpn2I8gxI+VlJWSrJZYMZQTFh8J7bwm6/Hh3VdY0agT5/RiMWerQhKmgVCwa+05Tn4wPINVuZFe5lnUlrK9I/bIhrez2thxtuILHHiuPRrAcI5cjUczdk04bPedKGA/42MqmmTVxGgYng+rW1OSJb4MictwVYpHXKxxI32CNyY5h/XKCS+4Evk0rk6WNj3Hwf2ATLXBUIGt7cl+9DzqBvxlvCdJxal9SQpM9CZJGSDAhm32GB13QBEaXzE/xC52bgMavBL5XDOPQv4c21CVJLPIMyUFMEApdse+Ke198iCh98GvkH/3ldkJh+CQbgBJUOizfIgv8q35ynHNOUQBY4UknoCztgrZP3DXdpyvTPrUb/dejZHz9yZMtUplJ570aQ4dw0WaOxKHlzvje0qpaYT0gthpyieKDzlH0rwF0EPZ3AryyHMPgKeukrnD7giw/UcuX3rD7yFs76DJebIliSJTp8qgAIp3uvhG/E6e31dECHTLmEPC6yC7hOEHJ+Oe4no9OGyyy688/ryLS5ymko+fGaG9I0g8h8xy3OskvpG37r4CSTj5xFnzch51+Yxq5svs+ZY0iuZHoyhzvDSK5iFwh+BokGebTvwIIEG01/dp7xXBJrjVjpS6472lSlPHpcC8H+cRu6HcYaPDLoi34m6vKCa6Ku46w4k273J+NAoEbB+cpd5gpNGgEifUzXhwUGJ/MfPavAtS64zdN4xc1L376tOQ4QR1r/mzlhkxJ6krHoCCTB+VOG3d/S+4Azr6+ONRXAll1YKEmnosDrWoi+Kl5w3i+dLWyAqyvVuiWQFS+rWs1+up9c3wc/saBhrOFuv1girLlVe419/x7bB9jZ223QubLKC8LO95Nn2pvoUTcHxFruG1F36G10+MULCTVCt2iOuTJhbDmp7FokFmqpeqHLnw0n15UH65pwuTwfvEaGmdT6IOK1bXeVfTS0CqEWmyfqen2JratIVMbgcQySyRFiCM9JpKpQ4XCvCjVJVXrJ/gIAM1I0ZhTTYMQ+c41TXZKHHxNRgpKTXuyK90svJVOA5MAtczIhK/YTEekExxUqRQgFJ+h7oFwwtDr9fQ+p934vyjwXZY2F2akTgDngpj610RSJsqpG5VNNsoV227UlVsUy3X7bJdxzWd1GzdkHgaEtIe8++aKtflek3TpFm8ths5fAHLuAMRz5LY07JuUYinJJ6xJ0+GhGRWz9juGyzR/0o8OH/GS7DIbIrIPLghIIkHLCEuTw2uorZYfh0aj9KobkyP6ZqlO2tZuh1f7B9ma8au3KcXuz6qVU484vBM5LWRIxLcAXRLvncz2Bsa+BrfAR2xb5ycJ37EF+8daTRK2J5rFeI5LLV+i49+j708TqjihDP0k1phGhIZBMTZu0873TC1ndEpdHxyO9YceOEfbbMLM05qXVKWXTTSeSWQYrR8bPy54+8gSx5V9rZveeZy2sJawzP6eDc1ydXH1R9e3e6+TlMIFgUSsUVhCY32hAw0qUPqoeO7UWItghBCGNVAVIUFojW+KOHWGyRkT9P+FkLQcqq77t1Xbk4xwPwZi2kWFdCZrYnb5w82LKWLcHn27A8TthVsZ1gUFhOgYVhIpinCHq6OltTkRMiYnBJihDANhloX9GdAesAYFE0d8kyYRWOzjCUbySRLOTfdV0T3iJ+2tTPWAt9JihvgScxkFjNorig8oSB+DZuex6FcEvv4+jsc/YYi+Fjs42voA0Lf9IiDfhud6b9DereqeSfaHjUDz3AyzqfmKcYuMcO7RrrXQAPX+gYdKtkjH9krn9hoERQ7stRm7ICMZg6TBTKX7FHgHRUxci3dfB/KMR+Tvatzsnd1zexdnc3e1VnsXX189q5+e+xdfRh7V6ex97EbEt8l4RuETfPuqwMBGbNYyR8rC/nnCj2/xxOJIL7NeR4FASVzovm7yKGQArz/cHB0eXn8lNE8seRL9QDgGKwpUNqJKZ1bFS5Ar+B1zLL8PLw4GF6EfddM8VUloXhtKsXHgzYB8mJMV0Hw1fUSvK4aFdNWaoZdtTViqnZF0c1KHcsavGoVcz0EX12O4KvrI/isEL0AwWdrtgKCz9JrbQRffYIEv3QU3wD+AoCvKOWKCpnz+Nq2IfwMwq+yhyRgkdkQ/qMD2ZDp5yT8ydj4UMJ/cERaYgNg9rxcaAcgMd9mB2Bhh1v1DkB1+R2A6j9zB0CbcwdAW/MOgDZ7B0CbtQOgPf4OgPbt7QBoD9sB0KbtAJxgFLCnn2HIeuxhpznB/pSaXeKgi6Pz86OLp8z1J4zJuYXgd2yjNcO5lsJ5rTwNzoXhJ9hcDMsq2FxbL5trhkwMrWJYcrlel2uaUVNsUzNwBSDcqOv19bC5thyba+tj86zguQCbZ2u2AjbP0mttbK49QTafFl83yL3QPfWqDpnq2KKzAe4M4NZ4El6bqsgGuNfGP0OmnxO4JwPeQ4F7PMwsdSP9/sm2EEUnNtlQ9MJetGqK1panaO2fSdG1OSm6tmaKrs2m6Nosiq49PkXXvj2Krj2MomvTKPoI0DPC8eeTAgQB1yImhOOA2YEAbbI0PJiTrd8R7BZOMUQL9P3Ho48nJ0eXl0/6sfaB9dI75D2BybEV2R31ITuumb9r5YS/69Vp/D0YmMn74+nwrYLDa+vl8Gq5XKsZsmobhk1MC+MyuFRdtauKiqv1irweDq8tx+G19XF4VvhdgMOzNVsBh2fptTYOrz1BDl8sQm/ofJEb4mUIlOzGW/a6tcH0DEyvsY/Fg9k2mP7ogDVk+jkxfTIePhTT54tCy9wEn3cuLkTxick2FL+wk62a4mvLU3ztn0nx9Tkpvr5miq/Ppvj6LIqvPz7F1789iq8/jOLrU++FS12v1yPIg5UWcnYKKabL/461SVwLVvMAWRGglNuZ9y75B4fyPwl0fvTLU+b3k5dqjVsOmHlgO3jDb5rH9oMjsCB7H7E76syKa+b4ejXheL02jeNhZCYAPh61VdB7fb30bqhV3SprhmIqRLNVUqmoBq7ZdkWtahVTXtMT7vXl6L2+PnrPCrsL0Hu2Ziug9yy91kbv9SdI74tH5g3BL0Lwsl4tQ/o7sl5tuD2D2/nfb4P1ZMPtj45UQ6afk9snI+GDb6/PHX+WYff7Z+FCxJ4Ya0PsC7vXqom9vjyx1/+ZxK7PSez6moldn03s+ixi1x+f2PVvj9j1hxG7Po3YDyHq9qlPw4g47EsjIQr3g1uz6zleh30Rypycfuj1qMvKo62g+GkbXeydHL9/ysR+yBk8sRwcsw+uCzIfsd+aCV2vxYSuKlM/hs7HYoLR0xGDyi/VOozaS1VfBbHra77fTipYBx+SNbVqm7qFsazLRrWmYFupaPqaiF1fjtj19RF7VthdgNizNVsBsWfptTZi158gsc8bmTecvuBHzyuQ9E5ZrzbEnkHseg3yelhfNsT+6Eg1ZPo5iX0yEj6U2GfFnyU/aD7XLFyI2BNjbYh9YfdaNbHryxO7/o8k9qo8H7FX5fUSe1WeSexQ5H5i5wUel9ir8jdH7FX5QcQOgzP98+a2R8U3IUQuQWbk8K9P6kFMp7D8BpBezEnt31Ps2uybtjx0zmYMfcrEzj+HDpZjmI75n4yL+N312H7skfnEgoK718zuEHdTdten3l3nwzIB74OBWwGxV+X1Enutpte1St3QbFU1cBmXYfU39ErZLGPdhDNrIXY2f5Yg9qHqqyb2zLA7P7FP0Wx5Ys/Ua13Eznrx5Ih9kci8ofYFqZ09k5uxXm2IPUMHReF5vb4h9sdGqmHTz0fsGZFwiY+wz4w/S1L7zFm4CLGnxtoQ+8LutWJiH8oqHkrsQyL+UcSuzEnsypqJXZlN7MosYlcen9iVb4/YlYcRuzL1+9nY9xcS1+Q5xDmBbAIddLE/J6S/x2EXgjdB7/f2j84vnzKin/mCmrmxkr/pDgYTZ+GYmW3dVK4nVK5OpXIxEhNUnozUKphcWS+T20qlUqlVsF2TNYKNKrzYVdOs1ohuVetVbT1MrizH5Mr6mDwrsC7A5NmarYDJs/RaG5MrT5DJZ8TeDYYviOEQGVoTK9IGwrMgXGepurqB8H8DJQ1MPyeET4a+B381W3bAWZK7Z0y7xag7ts6Guhf2p1VTt7I8dSv/TOpW56Rudc3Urc6mbnUWdauPT93qt0fd6sOoW51G3T94PW4KdPc//A/CIfHHPC7vvv7J8qy52PsMogMknCd754dHT5q8E2Oxh8EZIstwwIwGv7jZ4PelYOc/xa81U/h09hZjMcHeYqRWQd7qesm7bGqqocq6Vq+WKzLRq5qhy3LdlsuWrFVJdT3krS5H3ur6yDsruC5A3tmarYC8s/RaG3mrT5C854q/G/5e6EPmMr8BN7Yqbeg7Q4cNd/8bOGkR4p4MeQ8l7nsDzVKfJp8x3Rai7g1vL+RHqyZtdXnSVuchbRgatnL/EXkhmwXdcmwVJjJkayhQJumB3ajPYtDR4E1qD6iTi7M9kzhOH1sWdTtNabgLAxGjGWFOpITdkc7GjMKPTUNqHXgWKRjY542G3fvLQz9YjZDcVzSEVbbtUAPUhdIXUb/v+eF9FcBGfA0ZVDqBMw4N+Nn7agbEHK14IU4kdVqH4JaeSw0YoZAv6N2JVYBc869fhvhveg6zelOCDBtHZuSigW0H7pYb9UPmcPFovygUkA3Id3j8I+ph8CiGjm3bA3v5qFBopWwhTuVbuVzqD/G5th8EbD8hl8DKRAGHulftIOrz7ZBkXubH5yXDQb66cjbiRyCjlm+dQvbDvybSufvawQ4bdJhyYqqljU5p9YYYAQ0JaxkajpEoLmBQSAW8NtvAyMc6dcOw3yiVxJWwS/6ISLEHGAjNW4Q/M0SKtp9ubJyKSyyveMsvgrLjpzKUfTZN237PAE0TC8XK3NzcFDueB9QATaMQyI2EzbYBacBV6y3/QyTsWSmfmF0Cs5kgfE1M9D2vMNz2syEDsK/HZI3FAz3We9ZgQDtG0SVh2tVisciaCTwn4n+8FNwXWu4Do7Ov7mR/xpTisHv3N7NYsZhP9JSEolILWuPapG7Cw168zKLcqEeWYq9jHsm8kF9OPJG6oe8xM42cEKMpNtxY1MoYBxaXRukm32ePSozGzuQK284RTJqH9Ds6JL4HAM/29J7nEAKaa2V5cr61Z5oRoU6KWROl/5VnmTTELpS4Gbca92pWGIpDu8kmkhcF4P9RPrkkJMXWzQvrDo9fMO6+JsweaIxcw0vR9Hr51gE/gy7u/r72RbOponM2APPULFKzDw7Jj0tFC0bk2A3AWaIQHYAneA7/XByY/wyymQe1YkTFyHRu52rk5JZF0NE2GM5y648Nw/RWY+fPmPHZ8zp7gEsQudgcGfaK78WpoUpcvSHfS44ypgF3cDYP+JxJ/d7AIB9HbTU/uoIPVzU9N2TfCsSDeU7Uv38bnbo0bLOdza3tnWelUhpX/Bcv0s1wxPU0POtWaNwNe3zn4Hnuee7/AU4MDHiMvQAA"} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514659.22/warc/CC-MAIN-20240715040934-20240715070934-00042.warc.gz", "offset": 116903011, "length": 6579, "data": "H4sIAAAAAAAAAO0823LbtrbP1kz+AeHeMe0TixIly7YkU3tsWWmc+nZsp+1up8OBSEhCTJEsQfrSTr7nNN+Rrzl/cdYCSIrUxXHq9OlsJ5ZJEFg3LKwbQP14cNmvmUb9ReVHuKpeP4SsQyImwsAXLG08ojE0NuqN7Wp9t2q2rus7nXqr02j/nHa4ZE4QudXjow7ZTyK/kyTc7ZgN5g5HTrvaaNX3qtum2a4O63Wz6uzuObt0p95o0b3ei0o/8GPmx9UT5o/jSYeY7b12c9asCKJh6HGHxjzwa5M4DrtkKsYxPLLmSP2RRg73R8EcLaPdtsN2hu2quU3b1e1d4ITuNfeqlNXNJhu12q0W0iJBAGYniSKJPCgC2dtuOnsua1f3dswhMOQ2qpSyBvDXGO0093aHO62dDMjxRfXAdYE4ARzV20ajuW2YOztGIxc0jcYsrr6/PO4Q5Eh0arUhH3o8iCfst4QZUy4EsOuyUUR9hxmjqMbh7t4IJ+G/vFvPokk8CSJbMLbOXavZMM0U9EUUxIETeAowTO7ig9gT0N7MOObhhEXVq4TjPF+fXNn9twfwv1G3L85P/m026y37Cm5bOxkg+uAF1K0e8TETcYeICTU7F6ffvz84f9vf2f3h3c9nlyfvW2ffvzt6f/im0f9p8K51kI499ALnpjzy4N354Merw93TH/797v3p9fnb7Z/OB42fT/vfn1+1Ds7OMpm6MCd8xJmbE7CoHfeTeOq9vp96LyovKm+vry+Qf9DdOjn//kVFafJp4G8Rs0XeJZ7UapLpM/nu9Hpe9WJ2H9cQaJc4ExoJFltJPKruvaj8VO1H9M4DyV3DFIkRXAx8J3C5P+5A38S/Ya6E5jMHaeuQG8bCKvX4LajrDzR66JADx2FhnI+D7tSZMNTBOMKJCgMRV6HFubHqWySMWOFm1aMXlcF9yEH3OuSKxlukbpJ31EcZ1Em93pH/FadXoIH9ILjhwOfFJDx9OEQNrJ5fHPSPDi2HwprBuS/qZZcwBdx6E/Et0titvkv8KkixRUyz08ik2CWn9L56MGZWs65+5rC9vbgaXF0dH1l112kNKXPptuMM95xma7e53RptN9s7bDhsNtpdEtJ4YtXmxk+H5yF1qifn3x2fWS7zWMzcGW3XkwT5rgLfVbO9m/NtlmmbJyoFipSdHZwOrLRhdS+gf7tltnZa7b16e3m3o4Prg8ODq8FziDyhMNWnoCGo/F/U3ysW3bKoQ4KGuOOxM6leBHcsOgxQX26bRb3NNH2mtsNomUWub+/gcqrsvzw671//+2JAcEGQi/eHJ8d9olVrtR+b/Vrt6PqI/PT2+vSEwJqr1QZnWoXAj4ZmCMzb3d2dcdc0gmhcu75UC9U0cVB2bbixq/Uq+xI4LGFfWEvGmu12W43QsFPHo/7Y0kaRlq9PXS5QHSEx6vYqa/tTFlNpDaugw/zW0pyUR/QhMFDdWdqqxa6RWg7Gp1Nmacr6FoZenB6S76IgCWXfcucb9nAHDlIUuqfmnbiMKAO/RYrLbBGhy4QT8RANSQHMYTpmnY0jesu6MJSczkC/kaCXUBQFwyAu0kM9L8X5slo9HhAWEwEwmA9mDSE5wRTWIR9yj8f/W60uE6r+U/X9AeiU6ugxPYeuHw+sgTtm+hxPt5zdhUEUF+i44y6sdpfdcodV5c0W4T6POfWqwqEes8wtMqX3fJpMswbQmbW1NQAd89hjvT6NqReMQRLMJx4f+0tEsl9TfStr+G/f4/4NRD2epYv4wWNiwlisExlg6FInHCF0MonYyNKNmupTA5FMweOoPwb0+Je526jvtBvbO3VdyfI5cEcgkiq9YyKYsm8PPQiZXxXgtr496EhGg7bLRejRh28IP5wOt+2pOwJEHnrVMu27e98E9hhWAos4s8MJREt/C4rs4m+BHXE//jsAx3T8twjjluPK9Fki2HLwYI8IPLDR4mIAqAI9AjaoiPrq7fnldf/9NTnun59l2PiUQphZExDX1kYUTAqsU/iQZkhZU0Wmsvsf6C1VrRoRkWNp3He8BOxu4UnNjejY9m03CkLjg9B6+zX14BkgJ9R3PfZNQYZBmDwOTJ8DhjIhZI2PyMZLN3CSKYjagAxl4DG8FIcPfY8KcQZ2e/MPUqtB+JlEhDO0vF/oTyyCvUaJL8PgDT+Y2g4+3PwD29duKQCK2VRYPrsjB1FEHzY2u/kjJ0jAMdRVwyiIyIYcAC2E7y9HfU3HiHhD/y990/BkGAOdX7/eJH8QScuaZPQJg5GuDb5pODNmLDJjgCgOABzS/4sk9fXrX4Hjp8PuKoowUvqortSfiMWQeirJSOZlO/6C9N8ESpiEkiQGnyw4uGk5JZ7OMJoDxQdvJ8j7yxOBfnA205XZVBDZldnQacMFr6kmBMUL5PchAIKAtsgLxkTUiZls/9eK9k65HWIoST0fbRSBWkR7f/0GIqtUCVJuFUGQFWMYEfgARBEmQXxknmDl7hABhKzYRYlopvWVrzRPS9d88CGoufwDj2tggqbwME58MATpH0MCAFv1xQWmJPshgAh7xMcgWWQmRBmd+yeQz0K2GSVsCxohSYbQpkP0UVQdRTo2cXHEhgnE6CMKYsAWMJkXHuX+O4HJZTYUko4bNHsd8ouUFYZaAAgM7iEVTIKS4GVKqhtGTf7XUXC/4kOXhThWh6xamunaKRMCAV7FQcR02QeI84aAp5Mv6lInNad33HeDOwNAHAHPyO1HOUdpgzFNh8ATXPglCBt/JJEH1NXoB6pKHtPATSDYw/t1IJ6NrWy8vkVcSKic+BjixI6UzkepDcgSUAtYZ/rw+BxJi/qoFmQfYFwxwcjgfguwIJcc8lMJ7qFKBTDltpgEd7YIHAiUbZ/FkG/cELCRT7b/isZlFAo+BQtjC0YjZzJH3ZOALmccMwc7wmRHxPNQQbAqdavsDwP3gQQ+lnswO5Aq5bIRTbz4KqZxIiwdUy9ZSvn8pyBhAhmRo3c1QjiMAKGieLQnrM7cKo4Ay1kAkQZLLfcGd6VKo7ATz/ITz+tKO+xQIa2sL3sLIr0CR5YqmR2dKkjLXcLhw7G7oTNPf83d13p/wj1X31SgFcgsiq5IA/oyB7f5NMgKiAIpDXA2PDf0iSckgPTBCneVeApGyf8SdMAwPvWwRLlYZZ8BGTz5hf+KAA/iOOLDJAZAHPhD468H/thjsbC5GLp2OmevUzHnpAFlCopy/mvDiNEbdf0xt/Yk9wo4yOOPyUNhlUgltswzvfT4Jnny8AK5GYBsvKTZ4wYYdRh1Br6smzrt1GMlqDkfC8om1+3dhMYbvyUBh2TXVVGFqt3IRiUU0DVG9MFPFyf2yXlfJx0ph5W0uvzWZvehZ4Ot18lrhGtIB2ikKgWE6kMsxurdp0FaCcUHR613pb19BEwqvCKkQlRF9IIm2NyHcILffh3UIqcrIWdwK2V9mgn32whWieSZcn3S7HyNWEvMfzORziZrTqipmU4FippsEfw04uAES5J9EHox0peLZ6nx7VaymD3x8qA7zxuItBMrTJfHU9O1xHjJ0cvMl1zTKqb/kgmz9dfIU8F2ZdkAjHzaTHxJSWYYHteRUoQ8x8OMkpcz0jGn18n6Onm8G2TOT+kmkvH4Cd2GVNxABD6T1hPEVbAH+ahVIntk2oyIgdwclum4voXy1TcfX8CZt5n9/biWmqWPZY3/iKb9mwRHMQW+BYgew9+vCQifAPSbwZty4SzEgyBQDLx0GuNgFKul/149xq3Ljlmv17E39Cl0lcV4SGNYpPXytik0zG51vLUxJGQRAJg0e1nh+/OfWPJ2GfHosiov9FToSoDtSRAJexQEMeJM5/YZgsDSlJ3VXAriAJgZWo4benaasqSMSXWHELWhZUSWSNV9WHhjCukQZhoCxvvj3gGB0CLmqibP4pjBPaT6t0EiMP+/Zb+TDuBXnSv7OOWMqKK6DuJ/BdTFEfy6KXaFRNqjwAeZ8Ai0DshnzNdJb5+qxFgz5jaf5Z1WhgHXcRoIAxYRUr93yeIAoh4WEZrgFuUUyxSfPzkwO4TeModAPg6cRD7//Gc0i6MNwwAeEMB+jcJv7GKqhXRLHWaZoLA0OGLRFEMrqQT/yEWG9cFc+jrABlthA0QgQM/IZtMwftCLwP55DDn4mA5pxGMD94G2bolpNI2G0SSNurmD5x/qu6S+22ntdVq75EMI2KnvMzK4D8k/JVKcKtD6Ys3OEQLUzAA1yeJ5W00M2MC0AfJ3bOlWPlbKHaPgbrFbFVoXuzrM85b0xWawTiF1cY+vGvHxJO6QVniPEPZVQRbJzlRVgmSwIj0hFwcuNzWdR58/YbNcbjp0Y0mUzxN0KizpAgy7sLwRWrGLYCxTovxersdUcdNH8sYudYAeUQYGVjOPH+bwyC5utriLWNPupZkoUVF+0ss9TnHRKjgeu2Veo9BlTYkL7g+keNbWLg++G5xfk2uDZJ1SWRXHPPlnjoSSlsg1CTOMgsBTBTB7RSSzy8wALz55CpoMuo1bgoFg7lPRzP1Uyh2KN2rFp1eRUoh05SP8Utfc3xQ0Dr2c3GxcK2hviZ9iRzvmcQTzfJQGFGCROB4qwj1MIa0Vh7YQS9EsJkvUvmRmvJgCNGeSEVHCDTE19d0Dz9Ny06rP/Egnfyz9/oa52VVVnWIBxqaet5E+kCH6bcAFuLTSU0DLp+PMzsGlHXqJwN+ZH1N7IwolDjPGfKSTYRCBPbb0ui45Uo/hqZ6aYsV2iSe4cdilZP1KEqH11v2hCLvLejuB59FQsJUyKHTY2FzkYxpwX8iPeU6ygSt5yTo8l5u/GiUI5jFnodSltLh8K1MbVAF7jHF1EMVya6RYMcO8Keud1xKy3na6nb7hMk82YFmhsFuRplsR5luP11hx8LpIhlYGej2ivyuK9NcbKINglGPZlGF+gmfWwAy7Om4RpI9Ih+j65msdYIVhZAM1hfGStoWh0JqPwhjHZj7wlUTCggDkVfNodv7tVWOHu9CCZ+BmMfsaKYvQyCQHfG/Vt3R9y9xKi6YzicGD+mYh7F8Q8cKIjewEYlG8K9O5UQTZjZ3W9iAgwhMQuBUEIUSEJ2esMsl4gSq2MaNJTh0mbo8VqlagWZ4hp6qboSDqR6bL+VYb35f7v+leWjFRXssL/eyWehvYDZMvpHoOIsnTqNlFLtcb7ilNsSXtG/PKKh8hG9ZXMj0THJYPcjCbM/LztkLVDpRlGtwyWZEtDFpICBcXcsGWQO55Jdd8EElbF/h9SGZvLF0qEfhTFsHELq5asN+EerGlXUe85Iw0Ik+tLHuQZloOrA8IfvHgIGQcLMo8QdlUSrs4fLDp78pQUjwlY+nDII6DqU4m0gZaenPBhEoycRu9bEBBU6bqSI8+dWxQA1hnAFXOrLKGxdxhGkTMVq5S5CliFrPKvJtARD8JAB+eccTkh/thkpncCXddTE4UPuX2fvlVJxItNNjbeq33yAgpF7tuY8ds0OMjgnCu++C/Hx8w4sxz1ZgCZbTT6PwhOuZuZ/23JIi7gkUcJP87c1PnrZq7otMw0y60Y+KQnfT2anBw2X+bdjvrfswHmGYOU0YBSNQMWr38EMKJEMJpSaRIe33Ua4+KGQ1wzogpO9dw1p/qPFcHBHSEFj+yMWZiMIIVwoKi5hQ4s7GC5FpgYRg8zqU3xa1sAGfBr61cmZ4uGP1AYcEqQY5I7y1rLeq1LK6hgDA/Kvjp1XtI+AOBamqFxGLo9sujT1MD+6u1vbPX6FYWLEylGPJqzMNuF9Jo5QmMgl1VpizLlYoCz5C7LPQ4VdYjs0xpgIobxbgtheD1LbmzDHZqMRbd0GWPldFo/pykux9yc7arybMm6GKWRakqhtKMGthFsE5y8v1gCpcWPl4P5BlIn3mWqaXnCPl0cK8VRHI8Heem0lU5q5aZMq2uZQZOa+KRx9w+arl9RAtnU9vcuTd30EJq0hzrWYKgDoUzAuso4tN1Rh3ICbq5rn2hW23lnFSxwEU1gmeMxjgz+EDdqmKC6qY4VQeRoMFGjrXyohK4M4rFGBBROcxNSyyWmhmSCV+V1mzJgd77nrPoho0pjVw8CArLIwRHGIgA/jACSaZAPwX2w2NLlnQGTAVoEO2SGjn5/AmcfP/t4Or6/IdlY35utvf2chE68b19C5MFrcbeXrVRr2+v02nYjUYxuhd7NI0tfM3jVfMgGPFXjTfQANc37BY+p/E9fA6D4CYbA6GCHzGr1DRUqAqsvmrUWQwfHoWPAsPYXmQ5B/ENIKBdFVZzbxsH5M0uxN6W2W621pFhmO38SQih+DsDOv8QcX/WikVtCwwBF3kbTbAeZ6UiLzTLEp918qrffHXQDvz5AapeARiujflHV/23J+eDnweXC9AOA0ANY1ym5+pW+pMv+GFEsgUwy+LnLJqMuuYNmqMaVbigTWk05n5VRSydnfC+WwwgcPO/ZKjQZ5E4uGEQjoy2KUyHM2q4zbbj7Ow2GvVhy3T32numydo7VCco/pij2zN3G2a90Ww12lg3XVlLlXWtxJMh0pKNbbXUUkVftvGtzt6t4XnGAoxlo5fsIc3c5T+y9aNdyB5o1x1l12e7zLo6tQ9mfZVxVoOVH/R4OkNlypCMlXTlmzXLKDu+OjxaThc+eYQqfFymqaIyizVVHko8VSeaVa70KLib7TYUSkW4p1aU7VwIKr1+N5twnOMMYJxbSCygp1GzrKfmghiOq+OIPZTrPjqo02+2M4Flo/eusdoki/RqfYD6oAqV+qeS/OvmWEKVFfMy9Q8hpE/PIx88ERaAszRMRXYdMs9P0fd9/rSKIFVNE8+iSFVasViwINNlgeSSd+fMdksnvTkfRXtbRIEmXfJUSFiBIL285qugXEfUTZyvhGS2dwCSNKwo7tz2zoNcJVnmms+S6sDlkuJFRV3FgFRaAVF0xkNrFziQXgnm5p1B0GflGzpLSH4AW/08TfD9z5+kdqoFJM84rlJQ9K+rKPGlX34WLcdTfLMFdyGXKCY4fBKuQi4gJXwW6jeQmNF4CVqzTZzpKrT4HlXyTKZPFIxFpcH92M//Q0ETvjgWNB5cO/UKYMhlIgRbRbisVD6L7D6NP38aA2ImVmnLSq2X2DON326ixr89Pjm/Ooc/A6ntMuRZRXy6aRM/T9ckRQRSHQcyGIgvv5aJjIqMD3OvDYxs00NkQE1Zb/DT8dX14Oz6+ODk+Op0QKrk9PxocHmW3hSc1BecUMgiEcjqw7OYPgumJIyCMFrJbtFvXq0HydTrQlJcICvdU6pkMYKMUItRwizOWVKm6n6VZya/lD3ir5AV/X/wT2pyDPWnmtaB/oILySa54Ee2iMyRliCQ9lVddom0eka6DL9gfVIVypD9JbM1B0PZrmWx5H8M138M17MMV3ZqaMGgyTpkZbb1jfZM7nLb7J5NwYLhC+gQ481uyhva6ekLsFvpqRFLn+V6RRDldLii8uHJrGfopae2VHXQGYLOBi6rDmkkkcaTx/sHuIXeh8/HusaQi9geH6KZ1XtXSYiR12MDstdfZoNO8H0bLmTrYyOF+iqHAjbVkI3pHaWbMvietEx7JguaxG6ZrxePYOWi2d7d2c6UoPxQykGqdflpmfUTfhux+S5LmGXkgoH8k3siwOaf0shfGLXA6CkdU8H9peQJHicSB6gUuLfEl/qTWVx1Fu0o363Kj6eRmeZWiqqbu+Al5xtKVYfF50vz/XKZvQSh9Cg/CpZ+Fsv/xUN5Q5mmTCR3DsPtpC+/XYLHywqn/mMmYhs3UDbwY7Oy9kelonYo8d7A9MOQuy2kR0zcq5S7oWAho3hDO0sgfogCmefg4T+QZkg+f/LY7edP5KWWv6mRwhlBpi7SHeRSUaWS7nAvQ7v/RLRDKr4aZd6GGwvydQh5WC9/l6pw+EhJG9/eKewx4uXq7cXFiAfMLKx5/CIAYZnrXloFsejcPiMJfJEMpzzGgzyKvvIsyZ3Ihc1UWQGVNj/bTI1xBzBKdy4X+uPO9Hx3uVsdJPhKmRStWTOXDfVBreaHYttyRFizne+NbXq+eZcFvvu1VI3z+3wNyas1tfdUejo7iKWOYXKfHB3/QOaP1+KRyLy8m5+4reRuKW2zI3lEEl/8UvXihQ74GqctklAdIlx9LnUUyNxdlqjlFcDY1XqnGIfDBBMPAiYVpUGAoBxwjnQF1js2xFfIETMgTjUz7aC+tMNGxdRSmtKvLPnCtznlG1SLX9fRW3KQeYHYtVXUhtMhrpdUQoXvTxkHwdhD1CSWXz5l2UPI+W96b9CVk8Rns01QdTj3OzmgiHutIACPMx+RZeeoy9wjQsHHQ8Nncc6qYRiIRgReIq0gOBfArF5bpgSsi8tpeqQbumoZnboiVO8BNklNriayCJwqJKmUNbKWah1qpDqYW9hokMextd78+Ww5Z+kJVAiElh8onysza6GpzXud4slutS+gTZmfHLEoSICVWOu9qBCy7/GlJ6y13oHjJIx7eb17ofc/NCx74pHpYfEgvNTqF/JbdxBvthsIlgX0P9GyRwpSKl1NSbc4fwtfRobmAZAxCC+E4QRTrdeXLeTq85+3kUKbE/pEBPhapsGdEL/ZDK9rhgszcuwLUJYkJpDaTAKPp6f7ZQr4V7AMEyNxvIcnITl5CPx5HLivIKU/Nw2rsabKv2TFL1/Xyye4ht+Phl9oVtCK71RTYZAkb95aL18GUsHlt2asFRfCkAJ8mtjz7yAUh+YHq+UyUuMfD3fwG3ts3KGGGGCtVsvtSvTyZe7miaQTX+9VFONxe7zCb5r6P7BygIeWUAAA"} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763517701.96/warc/CC-MAIN-20240721121510-20240721151510-00045.warc.gz", "offset": 104033233, "length": 5171, "data": "H4sIAAAAAAAAAO076VLbyNa/cVXeoeNbieAL8g4YgzxljJOQMstgJ5PM1JSqLbXtDtpGahmYqbzP5DnyNPctvnO6JVk2NiED8+8mgOVW99n67C390rnslqulyrPCL3ClD28D1iIhiwLfi1gyeEwFDNYqtYZe2dNr1WG10ao2W9WdX5MJl8zyQ1s/OW6Rwzj0WnHM7VZjb8eu71qwZlzf1RvV5r7etBs1fbdC92il2aw1LKv9rND1PcE8ofeZNxHTFqnu1Pbr82FFEA0Ch1tUcN8rT4UIDogbTQTcMpZI/YWGFvfG/jItzSZjgFCnO409vbE/svRmrVnVm7vVxs5eY2e0V0daJAjAbMVhKJH7eSDWeAwc7TV1q2aN9IZVpfqIVmy9Ph7tjVjdblZsOwVycqF3bBuIi4Cjyn6pVm+Uqru7pVomaBpOmNDfX560CHIUtcrlER853BdT9kfMSi6PImDXZuOQehYrjcMyh283pWAa/OTMHMPzBbeYafMocOjtS24b9Vq1kYC/CH3hW74DwGt3hoQTwZbXU355MGWhPog57vKwPzC7bzvwU6uYF+f9T9V6ZcccwNed3RQQvXV8auvHfMIi0SLRlFZbnX79488f3/R3Prz78OHX44/9+qf33W7/faP+c+/TcedTsvbI8a2rxZXvTt8Mh3vvfv7wqVG7PP/l7c9vB+8uhjvnw7cfOh/OMx07sWFH+JgzOyPgrm7cTIXrvLpxnWeFZ4W3w+EF8FkFza2QZwVbavEg9rZJrUrexY7UaJLqMnlzOnxWsBK1ExK0YDeijCAPiDWlYcSEEYux3nxWmNHwtkU6lsUCofc8y7e5N4Hl1JoyHYGEKOjAj4QOI9aVUdkmQchyX9bdelZgNwEHzQFaqdgmFaCVeshDhVQqLfmjaAVyAJV/xYHSi2lwenuE+qOfX3S6x0cGkMJ2ce/yWnVAEuDGcBoD7LoOctBBDjukut+qp3I4IKf0Ru9MmFGvqH9L2N5eDHqDwcmxYVcrlfqotk9Zza7aozpr7O3vNveoZdf2G7Vm/YAEVEyN8tJ6d3QeUEvvn785OTNs5jDB7GXaqjrwrVf39zK+q4u0LROVAEXKzjqnPSMZWD8L6N+t1fb26/X6HQ6TacedYeeoM+g9hkiHwla7oCGovN/VwIiFMxa2iF+LrrmwpvqFf83CIx/1ZQY2+1HvhvTaAYtNdZUl6tcio3CFP23sVCtoDoXD58fn3eGnix5BlSYX74/6J11S1MvlX+rdcvl4eEw+vh2e9gnYTLncOysWCPwroncC53R9fV26rpf8cFIeXipDq1ZxUXpdsoVdbBcOJXAwQS8yVqyt7u/vqxVFnNRyqDcxiuOwmFmYJk1MQ0iM2u3CxqHLBJVOUgcd5jOjmLdSWKi+GcV15lok5QyMR11mFGkspn6YW3pxekTehH4cyLmLk6/Y7TWEtyg3PXHOxGZEuedtkjezuwhtFlkhD9BJ5cAcJWtesklIZ+wAlpLTOejXEvQKikJ/5Is8PdRxEpzPdf2kR5ggEcBg3piFCMnyXbBDPuIOF//V9VVC1T7q7zsQ/NREh2kZdO2kZ/TsCdOWeJpxdh34ocjRcc1tsHabzSAy6fLLNuEeF5w6emRRhxnVbeLSG+7GbjoAOrOxsQGgBRcOa3epoI4/AUkwjzh84q0QyWFZzS1s4P9Dh3tXkLM4hhaJW4dFU8aERmR6oEmdsKJII9OQjQ2tVFZzyiASFyKG+ijBjJ+qe7XK7n6tsVvRlCwfA3cMItHpNYt8lz09dD9gnh5RL3p60KHM5dK04gnhB+6oYbr2GBA5vuexRdr3mk8CewKWwELOzGAK2c6/giK9+Fdgh9wT/wZgQSf/ijBmHC3TY3HEVoMHf0TghokeFxM4lagR8EF51IO355fD7vshOemen6XYuEshTSxHkJeWxxRcCtgp/JFuSHlTRaby+5/pjKrRIolCyyhyz3Ji8Lu5O2U7pBPTM+3QD0qfo2L7sKxuPALklHq2w54UZOAH8f3AtCVgKBNCNviYbD63fSt2QdQlqC96DsPL6Oi2C1lIdAZ+e+svUi5D+hmHhDP0vN+ZTwyCs8axZ+HObXq+a1p4c+svHN+ATJjADrmR4bFr0glDeru5dZDdsvwYAkNFDYz9kGzKBTBC+OFq1EM6QcSb2v9pWyVHpjEw+dWrLfIXkbRsSEYfsBjp2uRbJWvOjEHmDBDFAYBD+n+TpL569Ttw/HDYB4oizJS+qCv1ETIBhaOSjGRejuMvSP+1r4RJKIkFxOSIQ5iWW+JoMpsDxYdoF5H3l/0I4+B8pwvzrSByKjNh0iYUN1RtCIoXyO9CAjSANCDHC+ZE1BJMjv+0Zry1OA45lKSejzfzQA1SfD98DZlVogQJt4ogqGkxjfA9AKIIkyC+MCdii9MhAwhYfooS0VzrCz/onlbavP/ZL9v8MxdlcEEu3BSxB44g+ShJAOCrvmtgSrKffciwx3wCkkVmApTRudeHehTqxTBm2zAIRS6kNi2ijUN9HGo4xKNjNoohRx9TEAOOgMu8cCj33kGmNl8KRccVur0W+U3KClMtAAQO94hGTIKS4FEBWlqpVJY/Ggrud7xpswDXalAVSzddPmVRhAAHwg+ZJucAcc4I8LQyo16YpPb0mnu2f10CEMfAM3L7Re5RMlBykyVwBw1/AcLmX3HoAHVl+pmqhgWUPzEke/j9JRDPJka6XtsmNhRUljiBPLElpfNFagOyBNQC1rk+3L9H0qPeqwXpH3CuWGCkcJ8CLMglg/xQgtuoUj5suRlN/Wsz8i1IlE2PCag3rgj4yAf7f0XjKgoj7oKHMSNGQ2u6RN2DgK5mHCsHM8RiJxLLUEGwqnQrHI58+5b4HrZrsDqQKmWzMY0dMRBUxJGhYeklWynf/o5IEENFZGkHRUI4rAChoniKD7DOzCuOAcuZ7I0lnnuT21KlUdixY3ix4xxIP2zRSHpZ1UmLiIwKHFkqpH7UVZBWh4Sj2xN7U2OO9orbr7TulDu2tqVALzbnCtKBPs/AbT0MsgKiQEoHnC7PHH3sRBJAcmNNuIodBWMh/hIMwLA+ibBEhVjlnwEZ3PmN/44AO0KEfBQLAMSBP3T+mu9NHCYik0cj20z27FUi5ow0oExBUcF/YxQyeqWuv2TenmRRARc5/D55KKwSqcSWRqbnDt8iD16eIzcFkK6XNDu8BE4dVp1BLDtIgnYSsWLUnC85ZZN2ez2lYvOP2OdQ7Noqq1C9GzmohAK6xojW+3jRN/vnXY20pBzW0mrzmcluAscEX6+RVwi3JANgKVEpIFQbYTNVO3gYpLVQPAjU2oH0t/eASYSXh5TLqoiW0wSTe5BO8NmPQc1zuhZyCrewqE9z4T6NYJVIHinXB+3Oj4h1gfknE+l8s5aEmrjpRKCoyQbBvyXh97El2QWh5zN9aTwrne9BIc3ZYydLurO6gUg/scZ1OTxxXSucl1y9yn1Jm1Y5/fdcmKm9Qp5yviutBmDlw3bie0oyx3C/jixkyEs8zCl5Picda3qNvHxJ7p8GlfNDpkXxZPKAaSMaXUEGPpfWA8SV8wfZqnUiu2fbSiEDuVks1XFtG+Wrbd1vwGm0mX9+2Ujc0pdFjf+Crv1JkiNBge8IRI/p748khA8A+mTwXB5Zd/JBECgmXhoVuBjFamh/6id48NiqVioVnA1zclNlMx7KGBYW29mYCwPzrxp+NTElZCEAmNbbaeP729/Y8rYZceiqLi/MVOgWAJtTP4zMse8LxJns7SMEga0pM+255MQBMFO0HA/0zKRkSRiT6g4paq2YErlAquaB4U0olENYaUSw3pu0OwRSC8FVT54JweA7lPozP46w/p+xP0kL8KvJhUPcckZUU10D8b8A6kQIv3aCXSGR/sj3QCY8BK0D8hnzNNI+pKowLpaWjo7lt+IiDDylShJhwBIF1GtfMuFD1sNCQmM8onSxTfHtqwW7Q+iMWQTqceAk9Pi3v8N5Hl0qlYAHBHBYpvArbCy1kG6pwywVFLYGxyx0MbWSSvCfTGTYH8ykrwFs8BUmQAQCtJRs5gbiNtPHudhV0pywABVIKBuRG99voCmfCxtE3aVSLPV2KNiODbvBI8z78bw9DU95331NueDeJJmJrQFm35kvSZpb3sbc9tTEC5mDZtyqlBTY4u5EdQ5hU1WDFBujJjWruzfV3dKEj4uEOsLQjhMPq46xGeFuEHL3JaMWeNQDjcgjlO9OKytBptqupKujLdMiwXbqBLcUb6ivqtepphWl7aieKwyYyFZxEV6ERSDqXbVI2qnWyAmJNhmalBpJ5aC8iCmJ19qn374K6vphMPWj1I+M8cAJbHwB2uJqdfwHek7KpOe61IuZQ153+j+/761a9Wt9v9nMBGaJG3PGQgNGS82mXqtUGi+pGxyEY2HOQGfGrjDwEZQX9Y4/5i9qr2EArq/YDP664gb+jnz/Kl0Dgc4LmbEwNFKoTl906y86+3MOX9QqNv5xKPxJ+cxW/aNFEKc9Y38P+ABO8Pmenb2Gvj+Hif4pMmqNJiwOsmF8lsKQnCPXsMvZnQhPXyKjj8i8mAv4iOJwxrjjKMLmqCE7MLosHM8HMLoboPg8ysYgBQXHZKjNyY1KV2ekm6dlupN83OM5YY/4xAMP4AfS6qEIlma3oqZVqpdowaqaV7XdN/AoIwdj1eoV6WPmn7X/pMpVvJAzisSHOMWtKzCQrMDU1IG9tk20xJGQpCqVTbODYlstVk7X4cqjLVGGZKylK8vTVlF2Mjg6Xk0X3rmHKry9SFNB9cs3lAuMnbn7S4gK/et5opHzjJhO52WbJChJ4teS2fSBluw+bngKUGQeAzUgUYCQT6ZzBzua6JOQ3WqLps8E/8O0pqB3WnvIBcSpVqZpENcO8Tc/P5HkA9zTS28UBQct0oOcgpNgyh0/8uEDE6IITwLwmB1SIsyE8Dwfvoffvkb4ZJqM9gtkyOi6yO5tACn24/gFV45Ep6l6SvGyAPJx49vXdQSBz2Ux+txHUNRRMMhdGjKFXU51lKs3I8bwCblavVaHxOiOz6ftbaKAryOf2dVHkd6zOYJfoT7rKJeqFE3ZnHigXDpH4B/9ZpZdrSD3ltHwcaL2vG9f5fYrlZYHDus0ACPBOkos33EeRUkXACiV/5F9R7Sp4BpN3PQ+GBQEJJKGI2Bvm5wMBmekUttv6vXqzkeUKFnHiCdD4aNYOXHxeRlp0SuE2GiSYB3yiP/5OOf12g9dKlahrZIdYrnrt89+HGJw/2dlFHO51zlbp0B3EpC15ODZ0ePUiYpvXyc+ZinrqFmvVIh97kv2QavOOsBb56wrnYgqNh62vF6voj2/7Z2dn8IvPgr5wzCq1V2AcdkbvL+87HWHJ+dnOQhrJCjBcVlKPGJTJU3EhmIQihCH/bAkUyoyTqQwa3SoTFBCuXh70j8fnMNHjxz3BuRDp997fzkgOrnAaIlBsTsFdyg4FIaRy+5GxKTeLKSpBOZqC8nEPB1aSiWwgXTw7wZw8tti4PwdipEnC2OKuJL60NXHP4g0KY9puNkmMulfAVx6L3V5QBKfsjhj8+l99RYpJco+kn7msDxqpyTfdSmrssL/+ZP/+ZMf9idpu08RtVBQLgpAdWsSfmf4LKsQDIayduQEhIBNHMlHHDqm/Gbg4fqLeudF7TX8qC6PzuISdemf+Kyq72bDcHEhp1XhB2t5dVWqNEunv6p/pc/B5KVqpmAwx2K/BmYhrWL/JVAFeI2szDtNOz9r/V5xaZOnICXs04ATnSWXNU21oFKWtUX9Sd1y0rjFLq9UDPTKAossk90wF/wwPmgPqfP8S7YlsCZtj1rMcQJq4xPwEnOqXjkQi42AguoETOczAyfpTqtjM2uEWafN9BENJVIxvX++DyzCCsHumyqgjjIdPgJyYfYgDjAXvG9B+pjPfFEfnyviEVX13/qVkYowOWxpzajWtI85vjWlngeXJdv0jmmzGfO0fKs5E0291mxmyr9wU8pBGt3i3UXW+3wWsuUpK5hl5IKB/OMbGU5PaejdWXWH0VM6oRH3VpIXgfpKHKBSEHNjT+pP4t2SnnsqGIdlbfhcQVDIq+68X4vAZz4HvMljNuZCi+Xu/ZXNjYUHdRYhLNxaWPyAnrZ6tCgPYTNt1chbd8nL7i/0qNNGe2qCigZsoOXa+dwjxycfyPIxDfbzs8OU7OSmkJl9MmaGUVRUDxAl3fvlCfg4oBnFgTz3ued8Y+zLak02DOUVwNgrtk/xEWTfiwhkFBOIM5EMDyraZEjXYL1mI3wUGTED4qR/nUxQL3+Y+LBeMaEpefXlO+/0FTO/e+e1j/aKA7E7xG6sozZwR/jUVCKh3Hs4E9+fOIiaCPkKomGOHOpdtV+jqySxxwjkp5ALwq865HkjF+Rxb+QE4HDmIbL0PG6Re0QY8cmoBGEgY7VUKiGayHdimQeD8QJm9fgrJZAJ25wmR4MwtZjSqSlCtTZgk9RkaiI7ikmSTwqLGllOtA41Up4qFeYmp471iu3lcz65Z+pkEYPT6oPJpZ5lMagWl606f0KomsxFl3nxMQv9GFgRxfazAiGHDl95Uldsdywrhuw3a57emf2fIrbE8OhtlD9QlVr9TL69hXjToxY/jkD/42J6S0FKpFtU0s3v351XUi2wHkDGwH1HmIUU2105Qgbf/p6FCm1G6AMR4ON9JW4F+H4rXpdLNuzIiQdZmIghIQNN8B2enBLLmuSfYBnFpdhybh+EpH+LgXIRBzappfSXtmE91kT5V1j8artevcFlfE8WX2zNacUbNZRbJMnL6V56tcIMpILLty828oYwogCfxubyWXZ+aXbOL81Irb8//GDCaOLx3+bWwUa5nPmV8PnzLKoQSSc+Jqooxtf38ArfWPx/bYjDo5w+AAA="} diff --git a/tests/repackage/test_repackage_cli.py b/tests/repackage/test_repackage_cli.py index 74b797f..9020c0e 100644 --- a/tests/repackage/test_repackage_cli.py +++ b/tests/repackage/test_repackage_cli.py @@ -1,7 +1,8 @@ """End-to-end tests of `cdxt repackage` with the async WARC fetcher. The captures are the first records of a Common Crawl CDX file (a CDXJ file); the WARC -records are fetched from data.commoncrawl.org (or s3://commoncrawl with AWS credentials). +records are fetched from data.commoncrawl.org or s3://commoncrawl, which are mocked with +recorded records (see tests/conftest.py). """ import gzip import os @@ -12,21 +13,28 @@ from warcio.archiveiterator import ArchiveIterator from cdx_toolkit.cli import main -from cdx_toolkit.record_io import read_records, write_records +from cdx_toolkit.record_io import parse_cdxj_line, read_records, write_records -from tests.conftest import requires_aws_s3, TEST_DATA_PATH +from tests.conftest import InlineExecutor, requires_aws_s3, TEST_DATA_PATH + + +pytestmark = pytest.mark.usefixtures('mock_data_commoncrawl') CDX_PATH = TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz' METADATA_RECORD_PATH = TEST_DATA_PATH / 'repackage/url_list.txt' -def write_captures(tmpdir, n: int = 10, name: str = 'captures.cdxj') -> str: - """The first n captures of the CDX fixture.""" +def write_captures(tmpdir, n: int = 10, name: str = 'captures.cdxj', s3_env=None) -> str: + """The first n captures of the CDX fixture (with their offsets in the mocked s3://commoncrawl, if given).""" path = os.path.join(str(tmpdir), name) - with gzip.open(CDX_PATH, 'rt') as f_in, open(path, 'w') as f_out: - for _ in range(n): - f_out.write(f_in.readline()) + with gzip.open(CDX_PATH, 'rt') as f_in: + lines = [f_in.readline() for _ in range(n)] + if s3_env: + write_records([s3_env.capture(parse_cdxj_line(line)) for line in lines], path, 'cdxj') + else: + with open(path, 'w') as f_out: + f_out.writelines(lines) return path @@ -45,13 +53,14 @@ def assert_repackage( base_prefix, extra_args: Optional[List[str]] = None, warc_filename: str = 'TEST_warc_by_index-001.warc.gz', + s3_env=None, ): base_prefix = str(base_prefix) main( args=[ '-v', 'repackage', - write_captures(tmpdir), + write_captures(tmpdir, s3_env=s3_env if warc_download_prefix.startswith('s3://') else None), '--no-sort', # keep the order of the CDX file '--add-metadata-record', str(METADATA_RECORD_PATH), @@ -108,7 +117,10 @@ def test_repackage_over_http_with_parallel_readers(tmpdir): assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=tmpdir, extra_args=['--parallel-readers=3']) -def test_repackage_over_http_with_processes(tmpdir): +def test_repackage_over_http_with_processes(tmpdir, monkeypatch, mock_data_commoncrawl): + if mock_data_commoncrawl is not None: + # the HTTP mock only applies in this process + monkeypatch.setattr('cdx_toolkit.repackage.multiprocess.ProcessPoolExecutor', InlineExecutor) assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=tmpdir, extra_args=['--processes=2']) @@ -179,19 +191,20 @@ def test_repackage_no_input_files_found(tmpdir, capsys): @requires_aws_s3 -def test_repackage_over_s3(tmpdir): - assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir) +def test_repackage_over_s3(tmpdir, s3_env): + assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir, s3_env=s3_env) @requires_aws_s3 -def test_repackage_over_s3_to_s3(tmpdir, s3_tmpdir): - assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=s3_tmpdir) +def test_repackage_over_s3_to_s3(tmpdir, s3_tmpdir, s3_env): + assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=s3_tmpdir, s3_env=s3_env) @requires_aws_s3 -def test_repackage_over_s3_to_s3_with_parallel_readers(tmpdir, s3_tmpdir): +def test_repackage_over_s3_to_s3_with_parallel_readers(tmpdir, s3_tmpdir, s3_env): assert_repackage( - tmpdir, 's3://commoncrawl', base_prefix=s3_tmpdir, extra_args=['--parallel-readers=3', '--is-part-of=foobar'] + tmpdir, 's3://commoncrawl', base_prefix=s3_tmpdir, extra_args=['--parallel-readers=3', '--is-part-of=foobar'], + s3_env=s3_env, ) @@ -201,5 +214,6 @@ def test_repackage_over_http_to_s3_with_parallel_readers(tmpdir, s3_tmpdir): @requires_aws_s3 -def test_repackage_over_s3_to_local_with_processes(tmpdir): - assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir, extra_args=['--processes=2']) +def test_repackage_over_s3_to_local_with_processes(tmpdir, s3_env): + # the worker processes use the mocked S3 too (it is configured with environment variables) + assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir, extra_args=['--processes=2'], s3_env=s3_env) diff --git a/tests/repackage/test_sql_index_e2e.py b/tests/repackage/test_sql_index_e2e.py new file mode 100644 index 0000000..581da99 --- /dev/null +++ b/tests/repackage/test_sql_index_e2e.py @@ -0,0 +1,131 @@ +"""End-to-end tests of the SQL indexes (DuckDB, Athena): `cdxt index ... | cdxt repackage`. + +By default, DuckDB queries a local columnar index (the local_url_index fixture) and the WARC records are +served by the mocked data.commoncrawl.org (see tests/conftest.py). With CDXT_LIVE_TESTS=1, the live tests +query Common Crawl's index on S3 (DuckDB) or with Athena, only a single crawl partition for a single host +(cheap, partition-pruned -- never an all-crawls scan), and fetch the WARC records over HTTP. +""" +import json +import os + +import fsspec +import pytest +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main +from tests.conftest import ( + LOCAL_INDEX_CRAWL, LOCAL_INDEX_HOST, TEST_ATHENA_S3_LOCATION, mock_response_from_jsonl, requires_aws_athena, + requires_duckdb, requires_live_aws, +) + + +CRAWL = 'CC-MAIN-2026-17' +HOST = 'commoncrawl.org' +ATHENA = f'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation={TEST_ATHENA_S3_LOCATION}' + + +@pytest.fixture(autouse=True) +def crawl_list_cache(tmp_path, monkeypatch): + """--crawl is resolved with the list of crawls; cache it apart from the list of other tests.""" + monkeypatch.setattr('cdx_toolkit.commoncrawl.CACHE_DIR', str(tmp_path / 'cache') + '/') + + +@pytest.fixture +def mocked(mock_data_commoncrawl): + """The mocked data.commoncrawl.org and the recorded list of crawls (for --crawl).""" + if mock_data_commoncrawl is not None: + mock_response_from_jsonl('test_get_cc_endpoints', 'test_cc') + + +def _index_and_repackage(tmpdir, index_args, host=HOST): + captures_path = os.path.join(str(tmpdir), 'captures.jsonl') + main(args=['index'] + index_args + ['-o', captures_path]) + + with open(captures_path) as f: + captures = [json.loads(line) for line in f] + assert len(captures) > 0, 'expected at least one capture for the host/crawl' + assert all(host in c['url'] for c in captures) + + base_prefix = str(tmpdir) + main(args=['repackage', captures_path, f'--prefix={base_prefix}/TEST_sql']) + + response_count = 0 + with fsspec.open(os.path.join(base_prefix, 'TEST_sql-001.warc.gz'), 'rb') as stream: + for record in ArchiveIterator(stream): + if record.rec_type == 'response': + response_count += 1 + target = record.rec_headers.get_header('WARC-Target-URI') or '' + assert host in target, f'unexpected target URI: {target}' + + assert response_count == len(captures), 'every capture should yield a response record' + return captures + + +@requires_duckdb +@pytest.mark.usefixtures('mocked') +def test_index_duckdb_local_e2e(tmpdir, local_url_index): + captures = _index_and_repackage( + tmpdir, [str(local_url_index), '--index-format', 'duckdb', '--crawl', LOCAL_INDEX_CRAWL, + '--hostname', LOCAL_INDEX_HOST], host=LOCAL_INDEX_HOST, + ) + # all captures of the host in the crawl, not those of the other crawl + assert len(captures) == 10 + assert {c['crawl'] for c in captures} == {LOCAL_INDEX_CRAWL} + + +@requires_duckdb +@pytest.mark.usefixtures('mocked') +def test_index_duckdb_local_domain_e2e(tmpdir, local_url_index): + captures = _index_and_repackage( + tmpdir, [str(local_url_index), '--index-format', 'duckdb', '--crawl', LOCAL_INDEX_CRAWL, + '--domain', 'missiondefrance.fr', '--limit', '5'], host=LOCAL_INDEX_HOST, + ) + assert len(captures) == 5 + + +@requires_duckdb +@pytest.mark.usefixtures('mocked') +def test_index_duckdb_local_sql_e2e(tmpdir, local_url_index): + sql = os.path.join(str(tmpdir), 'query.sql') + with open(sql, 'w') as f: + f.write(f""" + SELECT url, warc_filename, warc_record_offset, warc_record_length, content_languages + FROM read_parquet('{local_url_index}/crawl={LOCAL_INDEX_CRAWL}/subset=warc/*.parquet') + WHERE url_host_name = '{LOCAL_INDEX_HOST}' + LIMIT 3 + """) + captures = _index_and_repackage(tmpdir, [str(local_url_index), '--index-format', 'duckdb', '--sql', sql, + '--yes'], host=LOCAL_INDEX_HOST) + assert len(captures) == 3 + assert {c['languages'] for c in captures} == {'fra'} + + +@requires_aws_athena +def test_index_athena_e2e(tmpdir): + _index_and_repackage(tmpdir, [ATHENA, '--crawl', CRAWL, '--hostname', HOST]) + + +@requires_live_aws +@requires_duckdb +def test_index_duckdb_e2e(tmpdir): + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--hostname', HOST]) + + +@requires_live_aws +@requires_duckdb +def test_index_duckdb_domain_e2e(tmpdir): + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--domain', HOST, '--limit', '20']) + + +@requires_live_aws +@requires_duckdb +def test_index_duckdb_sql_e2e(tmpdir): + sql = os.path.join(str(tmpdir), 'query.sql') + with open(sql, 'w') as f: + f.write(f""" + SELECT url, warc_filename, warc_record_offset, warc_record_length, content_languages + FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl={CRAWL}/subset=warc/*.parquet') + WHERE url_host_name = '{HOST}' + LIMIT 10 + """) + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--sql', sql, '--yes']) diff --git a/tests/repackage/test_sql_index_gated.py b/tests/repackage/test_sql_index_gated.py deleted file mode 100644 index faa11af..0000000 --- a/tests/repackage/test_sql_index_gated.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Gated end-to-end tests for the SQL indexes (Athena, DuckDB): `cdxt index ... | cdxt repackage`. - -These query only a single crawl partition for a single host (cheap, partition-pruned --- never an all-crawls scan), write the captures with `cdxt index`, then fetch the -WARC records over HTTP with `cdxt repackage`. They require AWS credentials (and duckdb) -and are skipped in CI. -""" -import json -import os - -import fsspec -import pytest -from warcio.archiveiterator import ArchiveIterator - -from cdx_toolkit.cli import main -from tests.conftest import requires_aws_athena, requires_aws_s3, requires_duckdb, TEST_ATHENA_S3_LOCATION - - -CRAWL = 'CC-MAIN-2026-17' -HOST = 'commoncrawl.org' -ATHENA = f'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation={TEST_ATHENA_S3_LOCATION}' - - -@pytest.fixture(autouse=True) -def live_crawl_list_cache(tmp_path, monkeypatch): - """--crawl is resolved with the live list of crawls; cache it apart from the (mocked) list of other tests.""" - monkeypatch.setattr('cdx_toolkit.commoncrawl.CACHE_DIR', str(tmp_path / 'cache') + '/') - - -def _index_and_repackage(tmpdir, index_args): - captures_path = os.path.join(str(tmpdir), 'captures.jsonl') - main(args=['index'] + index_args + ['-o', captures_path]) - - with open(captures_path) as f: - captures = [json.loads(line) for line in f] - assert len(captures) > 0, 'expected at least one capture for the host/crawl' - assert all(HOST in c['url'] for c in captures) - - base_prefix = str(tmpdir) - main(args=['repackage', captures_path, f'--prefix={base_prefix}/TEST_sql']) - - response_count = 0 - with fsspec.open(os.path.join(base_prefix, 'TEST_sql-001.warc.gz'), 'rb') as stream: - for record in ArchiveIterator(stream): - if record.rec_type == 'response': - response_count += 1 - target = record.rec_headers.get_header('WARC-Target-URI') or '' - assert HOST in target, f'unexpected target URI: {target}' - - assert response_count == len(captures), 'every capture should yield a response record' - - -@requires_aws_athena -def test_index_athena_e2e(tmpdir): - _index_and_repackage(tmpdir, [ATHENA, '--crawl', CRAWL, '--hostname', HOST]) - - -@requires_aws_s3 -@requires_duckdb -def test_index_duckdb_e2e(tmpdir): - _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--hostname', HOST]) - - -@requires_aws_s3 -@requires_duckdb -def test_index_duckdb_domain_e2e(tmpdir): - _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--domain', HOST, '--limit', '20']) - - -@requires_aws_s3 -@requires_duckdb -def test_index_duckdb_sql_e2e(tmpdir): - sql = os.path.join(str(tmpdir), 'query.sql') - with open(sql, 'w') as f: - f.write(f""" - SELECT url, warc_filename, warc_record_offset, warc_record_length, content_languages - FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl={CRAWL}/subset=warc/*.parquet') - WHERE url_host_name = '{HOST}' - LIMIT 10 - """) - _index_and_repackage(tmpdir, ['cc-url-index-s3', '--sql', sql, '--yes']) diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 3292063..ca7e116 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -19,6 +19,7 @@ from cdx_toolkit.repackage.local_writer import LocalFileWriter from cdx_toolkit.repackage.records_source import RecordsSource from cdx_toolkit.repackage.warc_filter import WARCFilter +from tests.conftest import InlineExecutor from tests.fake_warc_server import FakeWarcServer from tests.conftest import requires_aioboto3 @@ -325,22 +326,6 @@ def test_no_s3_client_without_s3(tmp_path): session.assert_not_called() -class InlineExecutor: - """ProcessPoolExecutor stand-in that runs the workers in this process (patches apply, coverage is measured).""" - - def __init__(self, max_workers, mp_context, initializer, initargs): - initializer(*initargs) - - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def map(self, fn, items): - return [fn(item) for item in items] - - @pytest.mark.parametrize('remote', [False, True]) def test_multiprocess_repackage(tmp_path, remote): prefix = 'memory://mp-test/out/RUN' if remote else str(tmp_path / 'RUN') diff --git a/tests/repackage/test_warc_from_fs.py b/tests/repackage/test_warc_from_fs.py index aa23249..eb9da18 100644 --- a/tests/repackage/test_warc_from_fs.py +++ b/tests/repackage/test_warc_from_fs.py @@ -1,7 +1,10 @@ +import pytest + from tests.conftest import requires_aws_s3 from cdx_toolkit.warc import fetch_warc_record +@pytest.mark.usefixtures('mock_data_commoncrawl') def test_fetch_warc_record_from_http(): encoding = 'utf-8' capture = { @@ -28,7 +31,7 @@ def test_fetch_warc_record_from_http(): @requires_aws_s3 -def test_fetch_warc_record_from_s3(): +def test_fetch_warc_record_from_s3(s3_env): encoding = 'utf-8' capture = { 'url': 'https://bibliotheque.missiondefrance.fr/index.php?lvl=bulletin_display&id=319', @@ -45,7 +48,7 @@ def test_fetch_warc_record_from_s3(): } warc_download_prefix = 's3://commoncrawl' - record = fetch_warc_record(capture, warc_download_prefix) + record = fetch_warc_record(s3_env.capture(capture), warc_download_prefix) record_content = record.content_stream().read().decode(encoding, errors='ignore') assert record.rec_type == 'response' diff --git a/tests/test_cli.py b/tests/test_cli.py index e1f1508..3d56ef4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,8 +3,11 @@ import os import platform import logging +import re import pytest +import requests +import responses from cdx_toolkit.cli import main @@ -242,8 +245,14 @@ def test_multi_ia(capsys, caplog): multi_helper(testdata, capsys, caplog) +@responses.activate def test_multi_misc_not_ia(capsys, caplog): - # this is the test case before slow_ci -> break + # this is the test case before slow_ci -> break; the hostname does not exist (no DNS request: mocked) + dns_error = requests.exceptions.ConnectionError( + "HTTPSConnectionPool(host='web.arc4567hive.org', port=443): Failed to resolve 'web.arc4567hive.org' " + '([Errno 8] nodename nor servname provided, or not known)' + ) + responses.add(responses.GET, re.compile(r'https://web\.arc4567hive\.org/.*'), body=dns_error) testdata = ( '-v -v index https://web.arc4567hive.org/cdx/search/cdx --limit 10 --url commoncrawl.org/*', {'exception': ValueError}, diff --git a/tests/unit/test_myrequests.py b/tests/unit/test_myrequests.py index fede3cb..20a5e8e 100644 --- a/tests/unit/test_myrequests.py +++ b/tests/unit/test_myrequests.py @@ -1,10 +1,13 @@ import requests +import responses from unittest.mock import patch import pytest from cdx_toolkit.myrequests import myrequests_get +@responses.activate def test_get_without_error(): + responses.add(responses.GET, 'http://example.com/', body='') resp = myrequests_get('http://example.com') assert resp.status_code == 200 diff --git a/tests/unit/test_warc.py b/tests/unit/test_warc.py index 8a99fa5..2991f16 100644 --- a/tests/unit/test_warc.py +++ b/tests/unit/test_warc.py @@ -11,14 +11,14 @@ def test_wb_redir_to_original(): @requires_aws_s3 -def test_fetch_warc_record_from_s3(): +def test_fetch_warc_record_from_s3(s3_env): record = cdx_toolkit.warc.fetch_warc_record( - capture={ + capture=s3_env.capture({ 'url': 'https://bibliotheque.missiondefrance.fr/index.php?lvl=bulletin_display&id=319', 'filename': 'crawl-data/CC-MAIN-2024-30/segments/1720763514759.37/warc/CC-MAIN-20240716142214-20240716172214-00337.warc.gz', # noqa: E501 'offset': 111440525, 'length': 9754, - }, + }), warc_download_prefix='s3://commoncrawl', ) record_content = record.content_stream().read().decode(errors='ignore') From 0e7059e19903b4ac88f90f3969340cae48240b07 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 20:22:40 +0200 Subject: [PATCH 24/91] chore: CDXT_ prefix for all of cdx_toolkit's environment variables - tests: CDXT_DISABLE_MOCK_RESPONSES, CDXT_SAVE_MOCK_RESPONSES, CDXT_FAKE_GITHUB_ACTION - the log level: CDXT_LOGLEVEL (LOGLEVEL still works) --- .github/workflows/ci.yaml | 6 +++--- .travis.yml | 2 +- CHANGELOG.md | 1 + CONTRIBUTING.md | 8 ++++---- Makefile | 2 +- azure-pipelines.yml | 2 +- cdx_toolkit/cli/common.py | 2 +- tests/conftest.py | 24 ++++++++++++------------ tests/test_cli.py | 4 ++-- 9 files changed, 26 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 29aab8e..4df9dff 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,7 +39,7 @@ jobs: - python-version: '3.14' os: ubuntu-latest env: - LOGLEVEL=DEBUG + CDXT_LOGLEVEL=DEBUG EXTRA: true - python-version: '3.14' os: macos-latest @@ -94,7 +94,7 @@ jobs: core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_DISABLE_ATHENA_TESTS', '1') - core.exportVariable('LOGLEVEL', 'DEBUG') + core.exportVariable('CDXT_LOGLEVEL', 'DEBUG') - name: Lint code run: | @@ -153,7 +153,7 @@ jobs: core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_DISABLE_ATHENA_TESTS', '1') - core.exportVariable('LOGLEVEL', 'DEBUG') + core.exportVariable('CDXT_LOGLEVEL', 'DEBUG') - name: test minimal run: | diff --git a/.travis.yml b/.travis.yml index 6401533..f52c97e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: arch: s390x # unreliable in travis include: - python: 3.5 - env: LOGLEVEL=DEBUG # same time as normal + env: CDXT_LOGLEVEL=DEBUG # same time as normal - python: 3.8 arch: arm64 # 50% extra runtime - python: 3.8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c3de8e..031af69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ JSONL/CDXJ/CSV/TSV output, all fields by default, `--count` replaces `size` + `cdxt repackage`: reads captures from a file or stdin, human-readable `--size`, `--is-part-of`, `--description` + `--cc-sort` is now passed on to the CDX fetcher + + the log level variable is `CDXT_LOGLEVEL` (`LOGLEVEL` still works), like cdx_toolkit's other environment variables + `cdxt index` queries Common Crawl's columnar URL index with DuckDB (`cc-url-index-s3`, `cc-url-index-hf`, `*.parquet`) or AWS Athena (`athena://...`): `--hostname`/`--domain`/`--crawl`/`--limit` or your own `--sql`, cost confirmation (`--yes`), `--dry-run` + `cdxt repackage` copies WARC records byte-for-byte with the async WARC fetcher (from the feat/warc-range-sources branch): diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3bdd738..40fdf45 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,7 @@ make unit By default, all remote requests are mocked: the CDX API and HTTP downloads are served from recorded data in `tests/data`, and S3 is a local [moto](https://github.com/getmoto/moto) server (installed with `.[all]`; without it, the S3 tests are skipped). To change this behaviour and actually call remote APIs (if you run this from a whitelisted IP address), the following environment variable can be set: ```bash -export DISABLE_MOCK_RESPONSES=1 +export CDXT_DISABLE_MOCK_RESPONSES=1 ``` The S3 tests run against AWS, and the tests that need AWS and cannot be mocked (Athena, DuckDB queries of Common Crawl's index on S3) run at all, only with AWS credentials and: @@ -51,10 +51,10 @@ The WARC records that the `cdxt repackage` tests download (from `data.commoncraw If the remote APIs change, new mock data can be semi-automatically collected by setting another environment variable, running corresponding unit tests, and overwriting existing mock data in `tests/data/mock_responses`: ```bash -# set environment variable (DISABLE_MOCK_RESPONSES should not be set) -export SAVE_MOCK_RESPONSES=./tmp/mock_responses +# set environment variable (CDXT_DISABLE_MOCK_RESPONSES should not be set) +export CDXT_SAVE_MOCK_RESPONSES=./tmp/mock_responses -# run the test for what mock data should be saved to $SAVE_MOCK_RESPONSES//.jsonl +# run the test for what mock data should be saved to $CDXT_SAVE_MOCK_RESPONSES//.jsonl pytest tests/test_cli.py::test_basics ``` diff --git a/Makefile b/Makefile index b33e273..0e5b07a 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ clean_coverage: test_coverage: clean_coverage # PYTHONPATH=. coverage run -a --source=cdx_toolkit,examples examples/iter-and-warc.py # -rA to see all output, pass or fail - # LOGLEVEL=DEBUG + # CDXT_LOGLEVEL=DEBUG # -vvvv -s PYTHONPATH=. py.test -rA -s --doctest-modules --cov-report=xml --cov-append --cov cdx_toolkit tests -v -v coverage report diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ee56d4a..05173d7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ jobs: python.version: '3.6' Python36debug: python.version: '3.6' - myenv: LOGLEVEL=DEBUG EXACT_PIP_VERSIONS=1 + myenv: CDXT_LOGLEVEL=DEBUG EXACT_PIP_VERSIONS=1 Python37: python.version: '3.7' Python38: diff --git a/cdx_toolkit/cli/common.py b/cdx_toolkit/cli/common.py index 7245e3b..5afe303 100644 --- a/cdx_toolkit/cli/common.py +++ b/cdx_toolkit/cli/common.py @@ -33,7 +33,7 @@ def add_verbose_arg(parser: ArgumentParser, subcommand: bool = False): def set_loglevel(verbose: int): - loglevel = os.getenv('LOGLEVEL') or 'WARNING' + loglevel = os.getenv('CDXT_LOGLEVEL') or os.getenv('LOGLEVEL') or 'WARNING' # LOGLEVEL: the old name if verbose and verbose > 0: loglevel = 'INFO' if verbose and verbose > 1: diff --git a/tests/conftest.py b/tests/conftest.py index 3cefa0a..0d7eaa3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ By default, the tests make no remote requests: - the CDX API and HTTP requests (data.commoncrawl.org) are mocked with `responses` and recorded data - (DISABLE_MOCK_RESPONSES=1 sends them to the real servers, see mock_response_from_jsonl) + (CDXT_DISABLE_MOCK_RESPONSES=1 sends them to the real servers, see mock_response_from_jsonl) - S3 is a local moto server (see the s3_env fixture and requires_aws_s3) CDXT_LIVE_TESTS=1 runs the S3 tests against AWS instead, and enables the tests that need AWS and @@ -243,9 +243,9 @@ def _data_commoncrawl_range(request): @pytest.fixture def mock_data_commoncrawl(): """WARC record downloads from data.commoncrawl.org are served from recorded data (unless - DISABLE_MOCK_RESPONSES=1); other requests made with `requests` fail, unless the test adds mock data (e.g. + CDXT_DISABLE_MOCK_RESPONSES=1); other requests made with `requests` fail, unless the test adds mock data (e.g. with mock_response_from_jsonl).""" - if os.environ.get('DISABLE_MOCK_RESPONSES'): + if os.environ.get('CDXT_DISABLE_MOCK_RESPONSES'): yield None return responses.start() @@ -462,7 +462,7 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None (if you run this from a whitelisted IP address), the following environment variable can be set: ```bash - export DISABLE_MOCK_RESPONSES=1 + export CDXT_DISABLE_MOCK_RESPONSES=1 ``` If the remote APIs change, new mock data can be semi-automatically collected @@ -470,10 +470,10 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None and overwriting existing mock data in `tests/data/mock_responses`: ```bash - # set environment variable (DISABLE_MOCK_RESPONSES should not be set) - export SAVE_MOCK_RESPONSES=./tmp/mock_responses + # set environment variable (CDXT_DISABLE_MOCK_RESPONSES should not be set) + export CDXT_SAVE_MOCK_RESPONSES=./tmp/mock_responses - # run the test for what mock data should be saved to $SAVE_MOCK_RESPONSES//.jsonl + # run the test for what mock data should be saved to $CDXT_SAVE_MOCK_RESPONSES//.jsonl pytest tests/test_cli.py::test_basics ``` @@ -517,7 +517,7 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None def conditional_mock_responses(func=None, *, auto_mock_data: bool = True): - """Conditionally applies @responses.activate and auto-loads mock data based on DISABLE_MOCK_RESPONSES env var. + """Conditionally applies @responses.activate and auto-loads mock data based on CDXT_DISABLE_MOCK_RESPONSES env var. The mock data is automatically loaded from JSONL file from the tests/data directory and dependinng on the test module and test function. @@ -527,12 +527,12 @@ def conditional_mock_responses(func=None, *, auto_mock_data: bool = True): """ def decorator(f): - # If the flag DISABLE_MOCK_RESPONSES is not detected, response mocking remains enabled - if not os.environ.get('DISABLE_MOCK_RESPONSES'): + # If the flag CDXT_DISABLE_MOCK_RESPONSES is not detected, response mocking remains enabled + if not os.environ.get('CDXT_DISABLE_MOCK_RESPONSES'): # Add responses.activate f = add_mock_responses(f, auto_mock_data=auto_mock_data) - if os.environ.get('SAVE_MOCK_RESPONSES'): + if os.environ.get('CDXT_SAVE_MOCK_RESPONSES'): # Mock data is saved by capturing output from requests.get @functools.wraps(f) def wrapper(*args, **kwargs): @@ -607,7 +607,7 @@ def custom_behavior(*args, **kwargs): request_url=request_url, request_params=request_params, resp=response, - output_base_dir=os.environ.get('SAVE_MOCK_RESPONSES'), + output_base_dir=os.environ.get('CDXT_SAVE_MOCK_RESPONSES'), ) return response diff --git a/tests/test_cli.py b/tests/test_cli.py index 3d56ef4..f0633fa 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,8 +22,8 @@ def slow_ci(): For Github Actions, the windows and macos runners are very slow. Detect those runners, so that we can cut testing short. """ - if os.environ.get('FAKE_GITHUB_ACTION'): - LOGGER.error('limiting pytest because FAKE_GITHUB_ACTION') + if os.environ.get('CDXT_FAKE_GITHUB_ACTION'): + LOGGER.error('limiting pytest because CDXT_FAKE_GITHUB_ACTION') return True if os.environ.get('GITHUB_ACTION'): if platform.system() in {'Darwin', 'Windows'}: From 5bfdbbb6c7bdad29b810f643a39e29e2afdfc484 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 20:24:05 +0200 Subject: [PATCH 25/91] fix: file names of local paths on Windows (warcinfo WARC-Filename, one output per CDX file) The warcinfo record named the whole local path on Windows (split at '/' only), and one output per CDX file used the whole path as the output name; both use os.path.basename for local paths now. The S3 metadata test writes its file in binary mode (no CRLF on Windows). --- cdx_toolkit/indexes/cdx_files.py | 5 +++-- cdx_toolkit/repackage/warc_utils.py | 5 ++++- tests/repackage/test_warc_filter_pipeline.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cdx_toolkit/indexes/cdx_files.py b/cdx_toolkit/indexes/cdx_files.py index 6e00a7d..69cc8fd 100644 --- a/cdx_toolkit/indexes/cdx_files.py +++ b/cdx_toolkit/indexes/cdx_files.py @@ -196,8 +196,9 @@ def files(self) -> List[Tuple[str, str]]: import fsspec if not is_glob(self.location): - name = self.location.split('?', 1)[0] if '://' in self.location else self.location - return [(self.location, name.rstrip('/').rsplit('/', 1)[-1])] + if '://' in self.location: + return [(self.location, self.location.split('?', 1)[0].rstrip('/').rsplit('/', 1)[-1])] + return [(self.location, os.path.basename(self.location))] # also with backslashes on Windows fs, fs_path = fsspec.url_to_fs(self.location) first_glob = min(fs_path.index(c) for c in GLOB_CHARS if c in fs_path) base = fs_path[:fs_path.rfind('/', 0, first_glob)] diff --git a/cdx_toolkit/repackage/warc_utils.py b/cdx_toolkit/repackage/warc_utils.py index 566f7b2..e020a66 100644 --- a/cdx_toolkit/repackage/warc_utils.py +++ b/cdx_toolkit/repackage/warc_utils.py @@ -1,5 +1,6 @@ from io import BytesIO import logging +import os from pathlib import Path import fsspec from warcio.recordloader import ArcWarcRecord @@ -145,7 +146,9 @@ async def _start_with_header(new_writer, warc_file_path: str, writer_info: Dict, await new_writer.start() # Write WARC header; the `filename` field names this file - warc_file_name = warc_file_path.split('/')[-1] + # an S3 key or a local path (with backslashes on Windows) + warc_file_name = warc_file_path.split('/')[-1] if isinstance(new_writer, S3ShardWriter) \ + else os.path.basename(warc_file_path) buffer = BytesIO() warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) warcinfo = warc_writer.create_warcinfo_record( diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index ca7e116..a2975ce 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -276,7 +276,7 @@ def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): @requires_aioboto3 def test_s3_output_with_metadata_records(tmp_path): metadata = tmp_path / 'urls.txt' - metadata.write_text('https://example.com/\n') + metadata.write_bytes(b'https://example.com/\n') s3 = FakeS3Client() with patch('aioboto3.Session') as session, \ patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): From 3c9cbac6d649b9edc2fe847adeb990b3dcfe8eaa Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 20:36:31 +0200 Subject: [PATCH 26/91] test: spawn the worker processes of the mocked S3 test (a forkserver may have real AWS settings) Since Python 3.14, Linux starts worker processes from a forkserver, which an earlier test may have started with the environment of that time: the workers then read the real s3://commoncrawl (with CI's AWS credentials) instead of the moto server. Spawned workers get the current environment, which points to the moto server. --- tests/repackage/test_repackage_cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/repackage/test_repackage_cli.py b/tests/repackage/test_repackage_cli.py index 9020c0e..42400ba 100644 --- a/tests/repackage/test_repackage_cli.py +++ b/tests/repackage/test_repackage_cli.py @@ -5,6 +5,7 @@ recorded records (see tests/conftest.py). """ import gzip +import multiprocessing import os from typing import List, Optional @@ -214,6 +215,10 @@ def test_repackage_over_http_to_s3_with_parallel_readers(tmpdir, s3_tmpdir): @requires_aws_s3 -def test_repackage_over_s3_to_local_with_processes(tmpdir, s3_env): - # the worker processes use the mocked S3 too (it is configured with environment variables) +def test_repackage_over_s3_to_local_with_processes(tmpdir, s3_env, monkeypatch): + # the worker processes use the mocked S3 too: it is configured with environment variables, which a spawned + # process gets from this process (a forkserver, the default since Python 3.14 on Linux, could have been + # started before, with other environment variables, e.g. real AWS credentials) + spawn = multiprocessing.get_context('spawn') + monkeypatch.setattr('cdx_toolkit.repackage.multiprocess.multiprocessing.get_context', lambda *args: spawn) assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir, extra_args=['--processes=2'], s3_env=s3_env) From de2b3de901e771fac07b2afdccdf47af949d6dcd Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 21:10:20 +0200 Subject: [PATCH 27/91] ci: AWS login only in the slow CI, which runs the live tests The pull request CI needs no AWS credentials anymore (S3 is mocked); CI-slow logs in to AWS and runs the tests with CDXT_LIVE_TESTS=1 (and installs .[all] for the S3 and DuckDB tests). --- .github/workflows/ci-slow.yaml | 16 +++++++++++++++- .github/workflows/ci.yaml | 10 +--------- CONTRIBUTING.md | 2 ++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml index ae7f3c5..ac70953 100644 --- a/.github/workflows/ci-slow.yaml +++ b/.github/workflows/ci-slow.yaml @@ -2,6 +2,12 @@ name: CI-slow on: workflow_dispatch +# live tests (CDXT_LIVE_TESTS=1): S3, Athena and DuckDB against AWS. The permissions are needed to get AWS +# credentials via GitHub's OIDC token endpoint +permissions: + id-token: write + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.ref }} # 'queue: max' ensures old runs finish in order instead of being canceled @@ -42,8 +48,16 @@ jobs: pip install setuptools - name: Install cdx_toolkit - run: pip install .[test] + run: pip install .[all] + + - name: Configure AWS credentials from OIDC + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::837454214164:role/GitHubActions-Role + aws-region: us-east-1 - name: Run tests + env: + CDXT_LIVE_TESTS: '1' run: | make test_coverage diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4df9dff..957a8b9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,9 +8,8 @@ on: branches: - main -# These permissions are needed to interact with AWS S3 via GitHub's OIDC Token endpoint +# the tests make no remote requests (they are mocked); the live tests run in ci-slow.yaml permissions: - id-token: write contents: read pull-requests: read @@ -70,13 +69,6 @@ jobs: - name: Install cdx_toolkit run: pip install .[all] - - name: Configure AWS credentials from OIDC (disabled for forks) - if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push' - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: arn:aws:iam::837454214164:role/GitHubActions-Role - aws-region: us-east-1 - - name: Disable S3 unit tests for Python 3.8 (botocore requires Python 3.9+) if: ${{ startsWith(matrix.python-version, '3.8') }} uses: actions/github-script@v9 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40fdf45..d7de563 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,8 @@ The S3 tests run against AWS, and the tests that need AWS and cannot be mocked ( export CDXT_LIVE_TESTS=1 ``` +The CI workflow of pull requests runs the mocked tests only; the `CI-slow` workflow (started manually) runs the live tests with AWS credentials. + The WARC records that the `cdxt repackage` tests download (from `data.commoncrawl.org` or `s3://commoncrawl`) are recorded in `tests/data/repackage/warc_records.jsonl`, and DuckDB queries a small local index built from the same captures (the `local_url_index` fixture in `tests/conftest.py`). If the remote APIs change, new mock data can be semi-automatically collected by setting another environment variable, running corresponding unit tests, and overwriting existing mock data in `tests/data/mock_responses`: From 8c3cd9d1ea537704a844b0df0b6e6f17204a1b03 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 23:50:08 +0200 Subject: [PATCH 28/91] fix(repackage): a WARC file that cannot be closed is removed (final flush, rotation) --- cdx_toolkit/repackage/local_writer.py | 13 ++++++++---- cdx_toolkit/repackage/s3_writer.py | 3 ++- cdx_toolkit/repackage/warc_filter.py | 16 +++++++------- tests/repackage/test_local_writer.py | 3 ++- tests/repackage/test_warc_filter_pipeline.py | 22 ++++++++++++++++++++ 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/cdx_toolkit/repackage/local_writer.py b/cdx_toolkit/repackage/local_writer.py index f13aa25..e33df09 100644 --- a/cdx_toolkit/repackage/local_writer.py +++ b/cdx_toolkit/repackage/local_writer.py @@ -5,7 +5,9 @@ class LocalFileWriter: - """Async writer for local file system using aiofiles.""" + """Async writer for local file system using aiofiles. + + Like S3ShardWriter: close() completes the file, or removes it and raises; abort() removes it.""" def __init__(self, file_path: str, buffer_size: int = 8192, mode: str = 'wb'): self.file_path = file_path @@ -35,13 +37,16 @@ async def _flush(self): self.buffer.clear() async def close(self): + """Flush and close the file; if that fails, the (incomplete) file is removed.""" try: if self.buffer: await self._flush() - finally: if self.file_handle: - await self.file_handle.close() - self.file_handle = None + file_handle, self.file_handle = self.file_handle, None + await file_handle.close() + except BaseException: + await self.abort() + raise async def abort(self): """Close and remove the (incomplete) file.""" diff --git a/cdx_toolkit/repackage/s3_writer.py b/cdx_toolkit/repackage/s3_writer.py index ea82a65..1afa1a7 100644 --- a/cdx_toolkit/repackage/s3_writer.py +++ b/cdx_toolkit/repackage/s3_writer.py @@ -148,10 +148,11 @@ async def write(self, data: bytes): await self._upload_part(self.min_part_size) async def close(self): + """Upload the rest and complete the MPU; if that fails, the MPU is aborted.""" if self.failed: raise RuntimeError(f'the upload of {self.shard_key} failed, it was aborted') if self.buffer: - await self._upload_part(len(self.buffer)) + await self._upload_part(len(self.buffer)) # aborts on failure try: if self.parts: await mpu_complete( diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py index 7c2f044..74e8e77 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/warc_filter.py @@ -443,8 +443,8 @@ async def write_warc_records( tracker.start() counter = 0 - # Response records; any failure is fatal: the current file is aborted (removed, or its S3 upload - # aborted), the files written before stay complete + # Response records; any failure is fatal, also when closing a file: the current file is aborted + # (removed, or its S3 upload aborted), the files completed before stay try: while True: item = await warc_records_queue.get() @@ -470,8 +470,8 @@ async def write_warc_records( current_file_size + len(item.data) > self.max_file_size: # rotate due to the size limit (a file gets at least one record, also if the record is # larger than the size limit); the finished file is not aborted if starting the next fails - finished, writer = writer, None - await finished.close() + await writer.close() + writer = None current_file_sequence = self.next_file_sequence(current_file_sequence) writer, current_file_size = await self.start_file(current_file_sequence, **new_writer_kwargs) records_in_file = 0 @@ -485,15 +485,17 @@ async def write_warc_records( # Log progress every N items self.log_writer(counter=counter, tracker=tracker) + + if writer is not None: + await writer.close() except BaseException: + # also if closing failed: a writer that cannot close removes its file (or aborts its upload) itself, + # and abort() is a no-op then logger.error('WARC writer failed, aborting the current WARC file') if writer is not None: await writer.abort() raise - if writer is not None: - await writer.close() - return {'stats': tracker.get_stats()} def log_reader(self, reader_id: int, counter: int, tracker: ThroughputTracker): diff --git a/tests/repackage/test_local_writer.py b/tests/repackage/test_local_writer.py index 1e0650d..c8e5dce 100644 --- a/tests/repackage/test_local_writer.py +++ b/tests/repackage/test_local_writer.py @@ -54,7 +54,7 @@ def test_start_fails_without_the_directory(tmp_path): run(LocalFileWriter(os.path.join(str(tmp_path), 'missing', 'out')).start()) -def test_close_closes_the_file_if_the_last_write_fails(tmp_path): +def test_close_removes_the_file_if_the_last_write_fails(tmp_path): async def close_with_failing_flush(): writer = LocalFileWriter(str(tmp_path / 'out')) await writer.start() @@ -70,3 +70,4 @@ async def fail(): assert writer.file_handle is None and handle.closed run(close_with_failing_flush()) + assert not (tmp_path / 'out').exists() diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index a2975ce..1746e1b 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -124,6 +124,28 @@ async def failing_write(self, data): assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == ['out-001.warc.gz'] +@pytest.mark.parametrize('failing_close, expected_files', [ + (1, []), # rotation: the file being closed is removed, no next file is started + (2, ['out-001.warc.gz']), # the final close: the last file is removed, the complete first one stays +]) +def test_close_error_stops_the_pipeline_and_removes_the_file(tmp_path, failing_close, expected_files): + wf = WARCFilter(source=RecordsSource(CAPTURES[:2], 'https://data.example.org'), + prefix_path=str(tmp_path / 'out'), writer_info={}, max_file_size=1) + flushes = [] + original_flush = LocalFileWriter._flush + + async def failing_flush(self): + flushes.append(self.file_path) + if len(flushes) == failing_close: + raise OSError('disk full') + await original_flush(self) + + with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get), \ + patch.object(LocalFileWriter, '_flush', failing_flush), pytest.raises(OSError, match='disk full'): + wf.filter() + assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == expected_files + + def test_rotation_by_size(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, max_file_size=1) From b8496ef4b08047422e003f166a016c250521b2ce Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 23:53:09 +0200 Subject: [PATCH 29/91] fix(repackage): reject a prefix whose WARC series exists already (no overwriting an earlier run) --- README.md | 4 +++- cdx_toolkit/cli/repackage.py | 12 +++++++++++- cdx_toolkit/repackage/warc_utils.py | 18 +++++++++++++++++- tests/repackage/test_repackage_cli.py | 25 +++++++++++++++++++++++++ tests/repackage/test_warc_utils.py | 26 +++++++++++++++++++++++++- 5 files changed, 81 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index eb5afab..3af465a 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,9 @@ written to `[-]-001.warc.gz`, `-002`, ..., where the prefix can be a local path, `s3://...` or `hf://buckets/...` (default: `REPACKAGE` in the current directory, e.g. `--prefix out/CC-HOMEPAGES` writes `out/CC-HOMEPAGES-001.warc.gz`). No file is written when there are no -captures. The captures are +captures. If files of this series exist already (e.g. from an earlier run), +`cdxt repackage` stops before fetching (exit status 2): use another +`--prefix` or `--subprefix`. The captures are sorted by WARC file and offset first (for faster reads); `--no-sort` keeps the input order and streams large inputs without keeping them in memory. diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index 6d76955..ec890b5 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -25,7 +25,8 @@ Captures with a WARC location (filename, offset, length; e.g. Common Crawl captures) are copied byte-for-byte from --warc-download-prefix (https://, s3:// or hf://buckets/...) into -[-]-001.warc.gz, -002, ... (local, s3:// or hf://buckets/...). +[-]-001.warc.gz, -002, ... (local, s3:// or hf://buckets/...); files of this +series must not exist yet. Captures without a WARC location (e.g. Internet Archive captures) are replayed from a wayback machine with --wb and written to [-]-000000.extracted.warc.gz, ... @@ -172,6 +173,15 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): from cdx_toolkit.repackage.hf_utils import is_hf_url from cdx_toolkit.repackage.records_source import MissingWarcLocation, RecordsSource + from cdx_toolkit.repackage.warc_utils import existing_warc_files + + # every run numbers its files from 001: never overwrite (or mix with) the files of an earlier run + existing = existing_warc_files(args.prefix, args.subprefix) + if existing: + raise UsageError( + f'{len(existing)} WARC file(s) with this --prefix/--subprefix exist already, e.g. {existing[0]}; ' + 'use another prefix or subprefix, or remove them' + ) try: source = RecordsSource(records, args.warc_download_prefix, sort=not args.no_sort) diff --git a/cdx_toolkit/repackage/warc_utils.py b/cdx_toolkit/repackage/warc_utils.py index e020a66..f39979b 100644 --- a/cdx_toolkit/repackage/warc_utils.py +++ b/cdx_toolkit/repackage/warc_utils.py @@ -1,6 +1,9 @@ from io import BytesIO +import glob import logging import os +import posixpath +import re from pathlib import Path import fsspec from warcio.recordloader import ArcWarcRecord @@ -8,7 +11,7 @@ from warcio.recordbuilder import RecordBuilder -from typing import Dict, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import mimetypes @@ -83,6 +86,19 @@ def generate_warc_filename( return file_name +def existing_warc_files(prefix_path: str, writer_subprefix: Optional[str] = None, gzip: bool = True) -> List[str]: + """The files of the series [-]-NNN.warc[.gz] that already exist (local, s3://, hf://, ...).""" + fs, fs_prefix = fsspec.url_to_fs(prefix_path) + name = re.escape(posixpath.basename(fs_prefix)) + '-' + if writer_subprefix is not None: + name += re.escape(writer_subprefix) + '-' + name += r'\d{3,}\.warc' + (r'\.gz' if gzip else '') + return sorted( + fs.unstrip_protocol(path) for path in fs.glob(glob.escape(fs_prefix) + '-*') + if re.fullmatch(name, posixpath.basename(path)) + ) + + async def create_new_writer_with_header( sequence: int, output_path_prefix: str, diff --git a/tests/repackage/test_repackage_cli.py b/tests/repackage/test_repackage_cli.py index 42400ba..c53c602 100644 --- a/tests/repackage/test_repackage_cli.py +++ b/tests/repackage/test_repackage_cli.py @@ -191,6 +191,31 @@ def test_repackage_no_input_files_found(tmpdir, capsys): assert 'no files found' in capsys.readouterr().err +def test_repackage_rejects_an_existing_series(tmpdir, capsys): + """A second run with the same prefix would overwrite the first one's files.""" + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={tmpdir}/TEST']) + first = read_warc(os.path.join(str(tmpdir), 'TEST-001.warc.gz'))['response'] + + with pytest.raises(SystemExit) as exc_info: + main(args=['repackage', write_captures(tmpdir, n=2), f'--prefix={tmpdir}/TEST']) + + assert exc_info.value.code == 2 + assert 'exist already' in capsys.readouterr().err + assert read_warc(os.path.join(str(tmpdir), 'TEST-001.warc.gz'))['response'][0][1] == first[0][1] + assert not os.path.exists(os.path.join(str(tmpdir), 'TEST-002.warc.gz')) + # another subprefix is another series + main(args=['repackage', write_captures(tmpdir, n=2), f'--prefix={tmpdir}/TEST', '--subprefix=B']) + assert len(read_warc(os.path.join(str(tmpdir), 'TEST-B-001.warc.gz'))['response']) == 2 + + +@requires_aws_s3 +def test_repackage_rejects_an_existing_series_on_s3(tmpdir, s3_tmpdir): + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={s3_tmpdir}/TEST']) + with pytest.raises(SystemExit) as exc_info: + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={s3_tmpdir}/TEST']) + assert exc_info.value.code == 2 + + @requires_aws_s3 def test_repackage_over_s3(tmpdir, s3_env): assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir, s3_env=s3_env) diff --git a/tests/repackage/test_warc_utils.py b/tests/repackage/test_warc_utils.py index d34c6e0..3a21c94 100644 --- a/tests/repackage/test_warc_utils.py +++ b/tests/repackage/test_warc_utils.py @@ -1,4 +1,7 @@ -from cdx_toolkit.repackage.warc_utils import get_metadata_record_from_path +import fsspec +import pytest + +from cdx_toolkit.repackage.warc_utils import existing_warc_files, get_metadata_record_from_path from tests.conftest import TEST_DATA_PATH @@ -12,3 +15,24 @@ def test_get_metadata_record_from_path(): assert record_headers["WARC-Warcinfo-ID"] == "abc123", "Invalid Warcinfo-ID" assert record_headers["WARC-Payload-Digest"] == "sha1:VXA2A5YUS3TAY36AUO6MACRMNOH5RXG2", "Invalid payload digest" + + +@pytest.mark.parametrize('subprefix, expected', [ + (None, ['out-001.warc.gz', 'out-1000.warc.gz']), + ('SUB', ['out-SUB-002.warc.gz']), + ('OTHER', []), +]) +def test_existing_warc_files(tmp_path, subprefix, expected): + for name in ['out-001.warc.gz', 'out-1000.warc.gz', 'out-SUB-002.warc.gz', 'out-x.warc.gz', 'out-001.warc', + 'outer-001.warc.gz', 'other-001.warc.gz']: + (tmp_path / name).write_bytes(b'') + found = existing_warc_files(str(tmp_path / 'out'), subprefix) + assert [f.rsplit('/', 1)[-1] for f in found] == expected + + +def test_existing_warc_files_remote_and_missing_dir(tmp_path): + with fsspec.open('memory://existing/b[1]/out-003.warc.gz', 'wb') as f: + f.write(b'') + found, = existing_warc_files('memory://existing/b[1]/out') + assert found.endswith('existing/b[1]/out-003.warc.gz') + assert existing_warc_files(str(tmp_path / 'missing' / 'out')) == [] From 6c47ca8ac20c4dfbedf729f47cf3618c3dc91842 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 23:54:33 +0200 Subject: [PATCH 30/91] fix(index): exact --hostname for CDX files (www.), case-insensitive --hostname/--domain for SQL --- cdx_toolkit/indexes/cdx_files.py | 27 +++++++++++++++++++++++---- cdx_toolkit/indexes/patterns.py | 21 ++++++++++++++++++--- cdx_toolkit/indexes/sql_query.py | 6 +++--- tests/unit/test_selectors.py | 12 ++++++++++++ 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/cdx_toolkit/indexes/cdx_files.py b/cdx_toolkit/indexes/cdx_files.py index 69cc8fd..981d197 100644 --- a/cdx_toolkit/indexes/cdx_files.py +++ b/cdx_toolkit/indexes/cdx_files.py @@ -16,7 +16,8 @@ from .base import IndexBackend, UsageError from .patterns import ( # noqa: F401 (url_pattern_to_surts, to_surt) - domain_entry_to_surts, domain_to_surts, hostname_to_surts, to_surt, url_pattern_to_surts, + domain_entry_to_surts, domain_to_surts, hostname_to_surts, normalize_host, to_surt, url_hostname, + url_pattern_to_surts, ) from .query import IndexQuery @@ -58,6 +59,10 @@ class CaptureFilter: prefixes: Tuple[str, ...] = () exact: Tuple[str, ...] = () + # --hostname: the SURT prefixes select the candidates (with and without www.), the URL's host must be one of + # the hostnames + host_prefixes: Tuple[str, ...] = () + hostnames: Tuple[str, ...] = () select_all: bool = False from_ts: Optional[str] = None to: Optional[str] = None @@ -68,7 +73,7 @@ class CaptureFilter: def from_query(cls, query: IndexQuery, url_filters: bool = False) -> 'CaptureFilter': """The selection of the query; --url-fgrep/--url-fgrepv only with `url_filters` (one output per file: without --limit, they can be applied while reading; otherwise they are applied after the limit).""" - prefixes, exact = [], [] + prefixes, exact, host_prefixes = [], [], [] patterns = list(query.urls) if query.url_file: entries = load_url_patterns(query.url_file, query.url_column) @@ -83,12 +88,14 @@ def from_query(cls, query: IndexQuery, url_filters: bool = False) -> 'CaptureFil prefixes += p exact += e for hostname in query.hostnames: - prefixes += hostname_to_surts(hostname) + host_prefixes += hostname_to_surts(hostname) for domain in query.domains: prefixes += domain_to_surts(domain) return cls( prefixes=tuple(prefixes), exact=tuple(exact), + host_prefixes=tuple(host_prefixes), + hostnames=tuple(normalize_host(hostname) for hostname in query.hostnames), select_all=not (query.urls or query.url_file or query.hostnames or query.domains), from_ts=pad_timestamp(query.from_ts) if query.from_ts else None, to=pad_timestamp_up(query.to) if query.to else None, @@ -109,11 +116,23 @@ def __init__(self, capture_filter: CaptureFilter): self.prefixes = None if capture_filter.prefixes: self.prefixes = SURTMatcher(list(capture_filter.prefixes), match_subdomains=False) + self.host_prefixes = None + if capture_filter.host_prefixes: + self.host_prefixes = SURTMatcher(list(capture_filter.host_prefixes), match_subdomains=False) + self.hostnames = set(capture_filter.hostnames) + + def selected(self, key: str, line: str) -> bool: + """Selected by --url, --hostname, --domain or --url-file.""" + if self.filter.select_all or key in self.exact or (self.prefixes and self.prefixes.is_in(key)): + return True + if self.host_prefixes and self.host_prefixes.is_in(key): + return url_hostname(parse_cdxj_line(line).get('url', '')) in self.hostnames + return False def matches(self, line: str) -> bool: key, _, rest = line.partition(' ') f = self.filter - if not f.select_all and key not in self.exact and not (self.prefixes and self.prefixes.is_in(key)): + if not self.selected(key, line): return False if f.from_ts or f.to: timestamp = rest[:rest.find(' ')] diff --git a/cdx_toolkit/indexes/patterns.py b/cdx_toolkit/indexes/patterns.py index dc467d5..5061c14 100644 --- a/cdx_toolkit/indexes/patterns.py +++ b/cdx_toolkit/indexes/patterns.py @@ -37,14 +37,29 @@ def url_pattern_to_surts(pattern: str) -> Tuple[List[str], List[str]]: return [], [to_surt(pattern)] +def normalize_host(host: str) -> str: + """A hostname or domain as in the index (url_host_name of the columnar index): lower case, no trailing dot.""" + return host.strip().lower().rstrip('.') + + +def url_hostname(url: str) -> str: + """The (normalized) hostname of a URL, '' if it has none.""" + from urllib.parse import urlsplit + + if '://' not in url: + url = 'http://' + url + return normalize_host(urlsplit(url).hostname or '') + + def hostname_to_surts(hostname: str) -> List[str]: - """All URLs of exactly this host.""" - return [to_surt(hostname.strip() + '/')] + """The URLs of this host, and of the host with or without www. (the SURT key drops www.); the exact host is + checked with url_hostname().""" + return [to_surt(normalize_host(hostname) + '/')] def domain_to_surts(domain: str) -> List[str]: """All URLs of the domain and its subdomains.""" - return url_pattern_to_surts('*.' + domain.strip())[0] + return url_pattern_to_surts('*.' + normalize_host(domain))[0] def domain_entry_to_surts(entry: str) -> List[str]: diff --git a/cdx_toolkit/indexes/sql_query.py b/cdx_toolkit/indexes/sql_query.py index 29f8acd..1a2e039 100644 --- a/cdx_toolkit/indexes/sql_query.py +++ b/cdx_toolkit/indexes/sql_query.py @@ -7,7 +7,7 @@ from cdx_toolkit.commoncrawl import normalize_crawl, get_cc_endpoints, match_cc_crawls -from .patterns import domain_to_surts +from .patterns import domain_to_surts, normalize_host logger = logging.getLogger(__name__) @@ -86,8 +86,8 @@ def build_where_sql( added -- the main lever for reducing scan cost. `conditions` are further (already escaped) predicates that are AND-ed. Engines differ only in their FROM clause (see build_sql).""" - url_host_names = url_host_names or [] - domains = url_host_registered_domains or [] + url_host_names = [normalize_host(h) for h in url_host_names or []] + domains = [normalize_host(d) for d in url_host_registered_domains or []] surt_prefixes = url_surtkey_prefixes or [] surts = url_surtkeys or [] if not url_host_names and not domains and not surt_prefixes and not surts: diff --git a/tests/unit/test_selectors.py b/tests/unit/test_selectors.py index e076d76..acaba12 100644 --- a/tests/unit/test_selectors.py +++ b/tests/unit/test_selectors.py @@ -61,7 +61,19 @@ def indexes(tmp_path_factory): (['--domain', 'blog.example.com'], ['https://a.blog.example.com/', 'https://blog.example.com/']), (['--domain', 'a.blog.example.com'], ['https://a.blog.example.com/']), (['--domain', 'example.com.au'], ['https://example.com.au/']), + (['--domain', 'Example.COM.'], [ + 'https://a.blog.example.com/', 'https://ablog.example.com/', 'https://blog.example.com/', + 'https://example.com/', 'https://www.example.com/w', + ]), (['--hostname', 'blog.example.com'], ['https://blog.example.com/']), + # exact hosts, also with www. (which SURT keys drop) + (['--hostname', 'example.com'], ['https://example.com/']), + (['--hostname', 'www.example.com'], ['https://www.example.com/w']), + (['--hostname', 'WWW.Example.com'], ['https://www.example.com/w']), + (['--hostname', 'www.example.com', '--url', 'blog.example.com/*'], + ['https://blog.example.com/', 'https://www.example.com/w']), + (['--hostname', 'www.example.com', '--domain', 'example.com.au'], + ['https://example.com.au/', 'https://www.example.com/w']), (['--url', '*.blog.example.com'], ['https://a.blog.example.com/', 'https://blog.example.com/']), (['--url', 'blog.example.com/*'], ['https://blog.example.com/']), ]) From 388a5289693255086065f19bcbde1a5f1a9202fd Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 23:54:59 +0200 Subject: [PATCH 31/91] fix(repackage): worker processes keep the order of their shard (--no-sort streams, no second sort) --- cdx_toolkit/repackage/multiprocess.py | 3 ++- tests/repackage/test_warc_filter_pipeline.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index f0836dd..12dbc01 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -79,7 +79,8 @@ def _run_worker(cfg: Dict) -> RepackageResult: """Worker entrypoint (runs in a child process): fetch one shard into the output WARCs.""" if cfg.get('uvloop'): os.environ['CDXT_UVLOOP'] = '1' - source = RecordsSource(read_records(cfg['shard'], 'jsonl'), warc_download_prefix=None, sort=True) + # the shard is in the order of the source (sorted, unless --no-sort), streamed + source = RecordsSource(read_records(cfg['shard'], 'jsonl'), warc_download_prefix=None, sort=False) wf = WARCFilter( source=source, prefix_path=cfg['prefix_path'], diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 1746e1b..ba60f9d 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -377,3 +377,17 @@ def test_multiprocess_repackage(tmp_path, remote): finally: if remote: fs.rm('/mp-test', recursive=True) + + +def test_multiprocess_repackage_keeps_the_order_of_the_source(tmp_path): + """Workers do not sort their shard again (--no-sort streams, a sorted source is sorted already).""" + captures = CAPTURES[::-1] + with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ + patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + multiprocess.run_multiprocess_repackage( + source=RecordsSource(captures, 'https://data.example.org', sort=False), n_processes=1, + readers_per_process=1, prefix_path=str(tmp_path / 'RUN'), writer_info={}, writer_subprefix=None, + write_paths_as_metadata_records=None, log_every_n=1, max_attempts=1, uvloop=False, + ) + records = response_records(glob.glob(str(tmp_path / 'RUN-*.warc.gz'))) + assert [r.rec_headers.get_header('WARC-Target-URI') for r in records] == [c['url'] for c in captures] From f7db9b0d51f9e802c5349c2d046934f3c962f880 Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 23:55:41 +0200 Subject: [PATCH 32/91] fix(cli): --processes and --parallel-readers must be positive (0 readers hung) --- cdx_toolkit/cli/common.py | 11 +++++++++++ cdx_toolkit/cli/index.py | 4 ++-- cdx_toolkit/cli/repackage.py | 8 ++++---- cdx_toolkit/repackage/warc_filter.py | 2 ++ tests/repackage/test_repackage_cli.py | 9 +++++++++ tests/repackage/test_warc_filter_pipeline.py | 5 +++++ 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/cdx_toolkit/cli/common.py b/cdx_toolkit/cli/common.py index 5afe303..c0860b2 100644 --- a/cdx_toolkit/cli/common.py +++ b/cdx_toolkit/cli/common.py @@ -55,6 +55,17 @@ def parse_size(value: str) -> int: return int(float(match.group(1)) * SIZE_UNITS[match.group(2).lower()]) +def positive_int(value: str) -> int: + """An integer >= 1 (argparse type), e.g. a number of processes.""" + try: + number = int(value) + except ValueError: + number = 0 + if number < 1: + raise ArgumentTypeError(f'invalid value {value!r}, use a positive integer') + return number + + def parse_fields(value: str): """Comma-separated list of field names (argparse type).""" return [f.strip() for f in value.split(',') if f.strip()] diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index a952dc2..34cf1e7 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -7,7 +7,7 @@ FORMAT_CHOICES, canonical_format, format_from_path, import_pyarrow, open_text, write_records, ) -from .common import add_verbose_arg, parse_fields +from .common import add_verbose_arg, parse_fields, positive_int LOGGER = logging.getLogger(__name__) @@ -146,7 +146,7 @@ def add_parser(subparsers): cdx_files = parser.add_argument_group('CDX files options') cdx_files.add_argument( - '--processes', type=int, default=1, metavar='N', + '--processes', type=positive_int, default=1, metavar='N', help='number of processes, with one output per CDX file (default: 1)', ) diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index ec890b5..507978f 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -12,7 +12,7 @@ from cdx_toolkit.utils import get_version from cdx_toolkit.warc import fetch_wb_warc -from .common import IncompleteError, add_verbose_arg, parse_size +from .common import IncompleteError, add_verbose_arg, parse_size, positive_int LOGGER = logging.getLogger(__name__) @@ -76,11 +76,11 @@ def add_parser(subparsers): help="fetch captures by replaying them from this wayback machine ('ia' = Internet Archive)", ) source.add_argument( - '--processes', type=int, default=1, + '--processes', type=positive_int, default=1, help='number of worker processes, e.g. the number of vCPUs (default: 1)', ) source.add_argument( - '--parallel-readers', type=int, default=1, metavar='N', + '--parallel-readers', type=positive_int, default=1, metavar='N', help='number of concurrent WARC readers per process (default: 1)', ) @@ -196,7 +196,7 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): return run_multiprocess_repackage( source=source, - n_processes=max(1, args.processes), + n_processes=args.processes, readers_per_process=args.parallel_readers, prefix_path=args.prefix, writer_info=info, diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py index 74e8e77..5f74939 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/warc_filter.py @@ -111,6 +111,8 @@ def __init__( next_file_sequence: Number of the next output file, given the current one (default: current + 1). """ + if n_parallel_readers < 1: + raise ValueError(f'n_parallel_readers must be at least 1, not {n_parallel_readers}') self.source = source self.prefix_path = prefix_path self.writer_info = writer_info diff --git a/tests/repackage/test_repackage_cli.py b/tests/repackage/test_repackage_cli.py index c53c602..571a2b2 100644 --- a/tests/repackage/test_repackage_cli.py +++ b/tests/repackage/test_repackage_cli.py @@ -191,6 +191,15 @@ def test_repackage_no_input_files_found(tmpdir, capsys): assert 'no files found' in capsys.readouterr().err +@pytest.mark.parametrize('option', ['--processes', '--parallel-readers']) +@pytest.mark.parametrize('value', ['0', '-1', 'x']) +def test_repackage_rejects_non_positive_counts(tmpdir, capsys, option, value): + with pytest.raises(SystemExit) as exc_info: + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={tmpdir}/TEST', option, value]) + assert exc_info.value.code == 2 + assert 'positive integer' in capsys.readouterr().err + + def test_repackage_rejects_an_existing_series(tmpdir, capsys): """A second run with the same prefix would overwrite the first one's files.""" main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={tmpdir}/TEST']) diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index ba60f9d..b7b87ab 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -391,3 +391,8 @@ def test_multiprocess_repackage_keeps_the_order_of_the_source(tmp_path): ) records = response_records(glob.glob(str(tmp_path / 'RUN-*.warc.gz'))) assert [r.rec_headers.get_header('WARC-Target-URI') for r in records] == [c['url'] for c in captures] + + +def test_at_least_one_reader(): + with pytest.raises(ValueError, match='n_parallel_readers'): + WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=0) From b63792413e8ff474a223f3f499edb0636a371f9f Mon Sep 17 00:00:00 2001 From: malteos Date: Mon, 12 Oct 2026 23:57:55 +0200 Subject: [PATCH 33/91] fix(repackage): S3 range reads retry the whole transfer (also reading the body), the body is always closed --- cdx_toolkit/repackage/data_classes.py | 21 ++++--- cdx_toolkit/repackage/s3_utils.py | 21 ++++++- tests/repackage/test_warc_filter_pipeline.py | 63 ++++++++++++++++---- 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/cdx_toolkit/repackage/data_classes.py b/cdx_toolkit/repackage/data_classes.py index 16ea3a0..44756eb 100644 --- a/cdx_toolkit/repackage/data_classes.py +++ b/cdx_toolkit/repackage/data_classes.py @@ -2,7 +2,7 @@ import time from dataclasses import dataclass -from cdx_toolkit.repackage.s3_utils import is_s3_url, parse_s3_uri, with_retries +from cdx_toolkit.repackage.s3_utils import RETRYABLE_ERRORS, is_s3_url, parse_s3_uri, with_retries from cdx_toolkit.repackage.hf_utils import is_hf_url from typing import Optional, Tuple @@ -98,16 +98,23 @@ async def ranged_get_bytes( if readers is None: raise ValueError('s3:// range read requires readers (none were provided)') s3_client = await readers.s3() - resp = await with_retries( - lambda: s3_client.get_object(Bucket=bucket, Key=key, Range=f'bytes={offset}-{end}'), + + async def transfer() -> bytes: + """The whole transfer is retried: the request and reading the body (closed in any case).""" + resp = await s3_client.get_object(Bucket=bucket, Key=key, Range=f'bytes={offset}-{end}') + async with resp['Body'] as body: + data = await body.read() + if len(data) != length: + raise RangeError(f'{self.url}: requested {length} bytes from offset {offset}, got {len(data)}') + return data + + return await with_retries( + transfer, op_name=f'ranged_get {bucket}/{key}[{offset}:{end}]', max_attempts=max_attempts, base_backoff_seconds=base_backoff_seconds, + retry_on=RETRYABLE_ERRORS + (RangeError,), ) - data = await resp['Body'].read() - if len(data) != length: - raise RangeError(f'{self.url}: requested {length} bytes from offset {offset}, got {len(data)}') - return data elif self.is_hf(): # read from a Hugging Face bucket (hf://) via the configured HF reader diff --git a/cdx_toolkit/repackage/s3_utils.py b/cdx_toolkit/repackage/s3_utils.py index 5eb9470..cddb3a5 100644 --- a/cdx_toolkit/repackage/s3_utils.py +++ b/cdx_toolkit/repackage/s3_utils.py @@ -4,7 +4,7 @@ from os import urandom try: - from botocore.exceptions import ClientError, EndpointConnectionError + from botocore.exceptions import ClientError, EndpointConnectionError, HTTPClientError, IncompleteReadError except ImportError: # pragma: no cover - S3 support is optional (cdx_toolkit[s3]) class ClientError(Exception): pass @@ -12,6 +12,20 @@ class ClientError(Exception): class EndpointConnectionError(Exception): pass + HTTPClientError = IncompleteReadError = EndpointConnectionError + +try: + from aiohttp import ClientError as AiohttpClientError # aiobotocore's transport +except ImportError: # pragma: no cover + AiohttpClientError = EndpointConnectionError + +# errors of a request or of reading its response that may be transient: timeouts, closed connections, interrupted +# or incomplete responses, S3 errors (e.g. 503 SlowDown; botocore retries those itself, too) +RETRYABLE_ERRORS = ( + TimeoutError, asyncio.TimeoutError, ConnectionError, ClientError, EndpointConnectionError, HTTPClientError, + IncompleteReadError, AiohttpClientError, +) + logger = logging.getLogger(__name__) @@ -31,13 +45,14 @@ def is_s3_url(url: Optional[str]) -> bool: return bool(url) and url.startswith('s3:/') -async def with_retries(coro_factory, *, op_name: str, max_attempts: int, base_backoff_seconds: float): +async def with_retries(coro_factory, *, op_name: str, max_attempts: int, base_backoff_seconds: float, + retry_on: Tuple[type, ...] = RETRYABLE_ERRORS): """Perform operation with retries and backoff.""" last_exc = None for attempt in range(1, max_attempts + 1): try: return await coro_factory() - except (TimeoutError, ClientError, EndpointConnectionError) as exc: + except retry_on as exc: last_exc = exc if attempt >= max_attempts: logger.error('%s failed after %d attempts: %r', op_name, attempt, exc) diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index b7b87ab..23a01a8 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -249,6 +249,12 @@ async def get_object(self, Bucket, Key, Range): start, end = (int(n) for n in Range[len('bytes='):].split('-')) class Body: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + async def read(self): return SERVER.files[Key][start:end + 1] @@ -321,24 +327,59 @@ def test_failing_metadata_record_removes_the_file(tmp_path): assert glob.glob(str(tmp_path / 'out-*')) == [] -def test_s3_reads_check_the_length(): - class Readers: - async def s3(self): - class Client: - async def get_object(self, **kwargs): - class Body: - async def read(self): - return b'short' - return {'Body': Body()} - return Client() +class FakeS3Readers: + """An S3 client whose responses have the given bodies (bytes, or an exception raised when reading).""" + + def __init__(self, *bodies): + self.bodies = list(bodies) + self.requests = 0 + self.closed = 0 + + async def s3(self): + readers = self + + class Body: + def __init__(self, content): + self.content = content + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + readers.closed += 1 + + async def read(self): + if isinstance(self.content, Exception): + raise self.content + return self.content + + class Client: + async def get_object(self, **kwargs): + readers.requests += 1 + return {'Body': Body(readers.bodies.pop(0))} + + return Client() + +def test_s3_reads_check_the_length(): job = RangeJob(url='s3://commoncrawl/crawl-data/a.warc.gz', offset=0, length=10) with pytest.raises(RangeError, match='requested 10 bytes'): - asyncio.run(job.ranged_get_bytes(1, 0, readers=Readers())) + asyncio.run(job.ranged_get_bytes(1, 0, readers=FakeS3Readers(b'short'))) with pytest.raises(ValueError, match='requires readers'): asyncio.run(job.ranged_get_bytes(1, 0)) +@requires_aioboto3 +def test_s3_reads_retry_the_whole_transfer(): + from botocore.exceptions import ReadTimeoutError + + readers = FakeS3Readers(ReadTimeoutError(endpoint_url='s3://x'), ConnectionResetError(), b'short', b'0123456789') + job = RangeJob(url='s3://commoncrawl/crawl-data/a.warc.gz', offset=0, length=10) + assert asyncio.run(job.ranged_get_bytes(5, 0, readers=readers)) == b'0123456789' + assert readers.requests == 4 + assert readers.closed == 4 # also the failed responses + + @requires_aioboto3 def test_no_s3_client_without_s3(tmp_path): with patch('aioboto3.Session') as session, \ From 8ab38bba9c95c77bbea970bf5135b7e60a1a956c Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 00:04:36 +0200 Subject: [PATCH 34/91] fix(repackage): cancelled HTTP range reads do not wait for their thread (daemon thread, retries stop) --- cdx_toolkit/myrequests.py | 25 +++++++- cdx_toolkit/repackage/data_classes.py | 39 +++++++++++- tests/repackage/test_warc_filter_pipeline.py | 62 ++++++++++++++++++++ tests/unit/test_cli.py | 2 +- 4 files changed, 122 insertions(+), 6 deletions(-) diff --git a/cdx_toolkit/myrequests.py b/cdx_toolkit/myrequests.py index 85c0e30..d3fa0bb 100644 --- a/cdx_toolkit/myrequests.py +++ b/cdx_toolkit/myrequests.py @@ -78,6 +78,18 @@ def reserve_fetch_time(hostname: str, now: float) -> float: return fetch_time +class RequestCancelled(Exception): + """The request was cancelled with its `stop` event (while waiting before the request or a retry).""" + + +def _sleep(seconds: float, stop: Optional[threading.Event]) -> None: + """Sleep, or raise RequestCancelled as soon as `stop` is set.""" + if stop is None: + time.sleep(seconds) + elif stop.wait(seconds): + raise RequestCancelled() + + def finish_fetch(hostname: str) -> None: """After a request: the next one may start the minimum interval from now (or later, if reserved).""" with _retry_lock: @@ -95,7 +107,12 @@ def myrequests_get( raise_error_after_n_errors: Optional[int] = None, raise_warning_after_n_errors: Optional[int] = None, retry_max_sec: int = 60, + stop: Optional[threading.Event] = None, ): + """GET with the minimum interval between requests to a host, and retries. + + With `stop`, the waits (before the request, between retries) end with RequestCancelled once it is set; a + request in progress ends with its timeout.""" t = time.time() if raise_error_after_n_errors is None: @@ -112,7 +129,7 @@ def myrequests_get( dt = fetch_time - t if dt > 3.1: LOGGER.debug('sleeping for {:.3f}s before next fetch'.format(dt)) - time.sleep(dt) + _sleep(dt, stop) if params: if 'from_ts' in params: @@ -133,6 +150,8 @@ def myrequests_get( retries = 0 connect_errors = 0 while retry: + if stop is not None and stop.is_set(): + raise RequestCancelled() try: LOGGER.debug('getting %s %r', url, params) resp = requests.get(url, params=params, headers=headers, timeout=(30.0, 30.0), allow_redirects=False) @@ -155,7 +174,7 @@ def myrequests_get( LOGGER.log(level, 'retrying after %.2fs for %d', retry_sec, resp.status_code) if resp.text: LOGGER.log(level, 'response body is %s', resp.text) - time.sleep(retry_sec) + _sleep(retry_sec, stop) retry_sec = min(retry_sec * 2, retry_max_sec) continue if resp.status_code in {400, 404}: # pragma: no cover @@ -187,7 +206,7 @@ def myrequests_get( if connect_errors > raise_warning_after_n_errors: LOGGER.warning(string) LOGGER.info('retrying after {:.2f}s for '.format(retry_max_sec) + str(e)) - time.sleep(retry_max_sec) # notice the extra-long sleep + _sleep(retry_max_sec, stop) # notice the extra-long sleep retry_sec = min(retry_sec * 2, retry_max_sec) except requests.exceptions.RequestException as e: # pragma: no cover LOGGER.warning('something unexpected happened, giving up after %s', str(e)) diff --git a/cdx_toolkit/repackage/data_classes.py b/cdx_toolkit/repackage/data_classes.py index 44756eb..a1d720c 100644 --- a/cdx_toolkit/repackage/data_classes.py +++ b/cdx_toolkit/repackage/data_classes.py @@ -1,4 +1,5 @@ import asyncio +import threading import time from dataclasses import dataclass @@ -10,6 +11,35 @@ from cdx_toolkit.repackage.http_range import RangeError, check_range_response +async def run_in_daemon_thread(func, *args, **kwargs): + """Run a blocking function in a daemon thread of its own (like asyncio.to_thread). + + Unlike asyncio.to_thread, nothing waits for a cancelled call: not the cancelled task, not asyncio.run() (which + waits for the threads of the default executor) and not the interpreter at exit.""" + loop = asyncio.get_running_loop() + future = loop.create_future() + + def set_result(result, exception): + if not future.cancelled(): + if exception is not None: + future.set_exception(exception) + else: + future.set_result(result) + + def run(): + try: + result, exception = func(*args, **kwargs), None + except BaseException as e: + result, exception = None, e + try: + loop.call_soon_threadsafe(set_result, result, exception) + except RuntimeError: + pass # the event loop is closed: nobody waits for the result anymore + + threading.Thread(target=run, daemon=True).start() + return await future + + @dataclass class ThroughputTracker: """Track throughput metrics for fetchers and consumers.""" @@ -134,8 +164,13 @@ async def transfer() -> bytes: headers = {'Range': 'bytes={}-{}'.format(offset, end)} # in a thread, so that the event loop (other readers, the writer) is not blocked; myrequests_get keeps - # the minimum interval between requests to a host across threads - resp = await asyncio.to_thread(myrequests_get, self.url, headers=headers) + # the minimum interval between requests to a host across threads. If this task is cancelled, the + # thread stops at its next wait (before a retry), and nothing waits for it. + stop = threading.Event() + try: + resp = await run_in_daemon_thread(myrequests_get, self.url, headers=headers, stop=stop) + finally: + stop.set() check_range_response( resp.status_code, resp.headers.get('Content-Range'), len(resp.content), offset, length, self.url ) diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 23a01a8..7777148 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -7,12 +7,15 @@ import itertools import os import threading +import time from unittest.mock import Mock, patch import fsspec import pytest +import requests from warcio.archiveiterator import ArchiveIterator +from cdx_toolkit import myrequests from cdx_toolkit.repackage import multiprocess from cdx_toolkit.repackage.data_classes import RangeJob, RepackageResult from cdx_toolkit.repackage.http_range import RangeError @@ -200,6 +203,65 @@ def main(): assert glob.glob(str(tmp_path / 'out-*')) == [] +def cancel_http_read_after_the_first_request(job, request_started): + """Cancel the read once the first request has started; returns how long asyncio.run() took after cancelling.""" + async def main(): + task = asyncio.create_task(job.ranged_get_bytes(1, 0)) + while not request_started.is_set(): + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + return time.monotonic() + + cancelled_at = asyncio.run(main()) + return time.monotonic() - cancelled_at + + +def test_cancelled_http_read_does_not_wait_for_the_request(monkeypatch): + """asyncio.run() returns while the (uninterruptible) request is still blocked.""" + monkeypatch.setitem(myrequests.retry_info, 'blocked.example.org', {'next_fetch': 0, 'minimum_interval': 0}) + started, release = threading.Event(), threading.Event() + + def blocking_get(*args, **kwargs): + started.set() + release.wait(timeout=10) + raise requests.exceptions.ConnectionError('released') + + try: + with patch('cdx_toolkit.myrequests.requests.get', side_effect=blocking_get): + job = RangeJob(url='https://blocked.example.org/a.warc.gz', offset=0, length=1) + assert cancel_http_read_after_the_first_request(job, started) < 1 + assert not release.is_set() + finally: + release.set() + + +def test_cancelled_http_read_stops_retrying(monkeypatch): + """The thread does not keep retrying (here: waiting 60s after a connection error) after the cancellation.""" + monkeypatch.setitem(myrequests.retry_info, 'down.example.org', {'next_fetch': 0, 'minimum_interval': 0}) + started, finished = threading.Event(), threading.Event() + requests_made = [] + + def failing_get(*args, **kwargs): + requests_made.append(args) + started.set() + raise requests.exceptions.ConnectionError('down') + + def get(*args, **kwargs): + try: + return myrequests.myrequests_get(*args, **kwargs) + finally: + finished.set() + + with patch('cdx_toolkit.myrequests.requests.get', side_effect=failing_get), \ + patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=get): + job = RangeJob(url='https://down.example.org/a.warc.gz', offset=0, length=1) + cancel_http_read_after_the_first_request(job, started) + assert finished.wait(timeout=2), 'the request thread is still waiting to retry' + assert len(requests_made) == 1 + + def test_hf_reads_use_the_hf_reader(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'hf://buckets/commoncrawl/commoncrawl'), prefix_path=str(tmp_path / 'out'), writer_info={}) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index af56e85..70758b9 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -72,7 +72,7 @@ def test_repackage_fetches_by_warc_location(tmp_path): main(['repackage', captures, '--prefix', str(tmp_path / 'out'), '--warc-download-prefix', 'https://x.org/']) assert get.call_args.args == ('https://x.org/crawl-data/x.warc.gz',) - assert get.call_args.kwargs == {'headers': {'Range': f'bytes=10-{9 + len(RECORD)}'}} + assert get.call_args.kwargs['headers'] == {'Range': f'bytes=10-{9 + len(RECORD)}'} with open(tmp_path / 'out-001.warc.gz', 'rb') as f: records = list(ArchiveIterator(f)) From 84822d4b0816e0c0249cc382b960c8672ecb1a53 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 00:06:34 +0200 Subject: [PATCH 35/91] refactor(repackage): one writer life cycle (start_file aborts, close aborts itself); remove unused content_type/writer_kwargs --- cdx_toolkit/repackage/s3_writer.py | 2 - cdx_toolkit/repackage/warc_filter.py | 59 ++++++++++---------- cdx_toolkit/repackage/warc_utils.py | 82 ++++++---------------------- tests/repackage/test_s3_writer.py | 2 +- 4 files changed, 46 insertions(+), 99 deletions(-) diff --git a/cdx_toolkit/repackage/s3_writer.py b/cdx_toolkit/repackage/s3_writer.py index 1afa1a7..012a486 100644 --- a/cdx_toolkit/repackage/s3_writer.py +++ b/cdx_toolkit/repackage/s3_writer.py @@ -92,7 +92,6 @@ def __init__( s3_client, shard_key: str, dest_bucket: str, - content_type: Optional[str], min_part_size: int, max_attempts: int, base_backoff_seconds: float, @@ -100,7 +99,6 @@ def __init__( self.s3_client = s3_client self.shard_key = shard_key self.dest_bucket = dest_bucket - self.content_type = content_type self.min_part_size = min_part_size self.max_attempts = max_attempts self.base_backoff_seconds = base_backoff_seconds diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/warc_filter.py index 5f74939..48c2bbc 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/warc_filter.py @@ -14,8 +14,9 @@ ) from cdx_toolkit.repackage.hf_utils import make_hf_reader from cdx_toolkit.repackage.data_classes import RangeJob, RangePayload, RepackageResult, ThroughputTracker -from cdx_toolkit.repackage.warc_utils import create_new_writer_with_header -from cdx_toolkit.repackage.warc_utils import get_bytes_from_warc_record, get_metadata_record_from_path +from cdx_toolkit.repackage.warc_utils import ( + get_bytes_from_warc_record, get_metadata_record_from_path, new_warc_writer, warcinfo_record_bytes, +) _STOP = object() @@ -78,12 +79,10 @@ def __init__( n_parallel_readers: int = 1, max_attempts: int = 5, base_backoff_seconds: float = 0.5, - # writer_kwargs: Optional[Dict] = None, range_jobs_queue_size: int = 1000, warc_records_queue_size: int = 200, aws_region_name: Optional[str] = None, warc_version: str = '1.0', - content_type: Optional[str] = None, min_part_size: int = 5 * 1024 * 1024, # 5 MiB (for upload) max_file_size: Optional[int] = 1 * 1024 * 1024 * 1024, # 1 GiB (for WARC outputs) next_file_sequence: Optional[Callable[[int], int]] = None, @@ -100,12 +99,10 @@ def __init__( n_parallel_readers: Number of async readers per process (one writer per process). max_attempts: Maximum retry attempts for failed operations. base_backoff_seconds: Base backoff time in seconds for retries. - writer_kwargs: Optional additional kwargs for writers. range_jobs_queue_size: Maximum size of range jobs queue. warc_records_queue_size: Maximum size of WARC records queue. aws_region_name: AWS region for S3 operations (default: the user's AWS configuration, else us-east-1). warc_version: WARC format version (e.g., '1.0' or '1.1'). - content_type: Optional content type for WARC output. min_part_size: Minimum part byte size for multipart uploads (default: 5 MiB). max_file_size: Maximum byte size for individual WARC files (default: 1 GiB). next_file_sequence: Number of the next output file, given the current one @@ -120,7 +117,6 @@ def __init__( self.write_paths_as_metadata_records = write_paths_as_metadata_records self.log_every_n = log_every_n - # self.writer_kwargs = writer_kwargs self.range_jobs_queue_size = range_jobs_queue_size self.warc_records_queue_size = warc_records_queue_size self.aws_region_name = aws_region_name @@ -136,7 +132,6 @@ def __init__( self.gzip = True self.warc_version = warc_version - self.content_type = content_type self.min_part_size = min_part_size self.max_file_size = max_file_size # Number of the next output file, given the current one (0 before the first file). The @@ -428,19 +423,6 @@ async def write_warc_records( current_file_size = 0 records_in_file = 0 - new_writer_kwargs = dict( - s3_client=s3_client, - output_path_prefix=self.prefix_path, - max_attempts=self.max_attempts, - base_backoff_seconds=self.base_backoff_seconds, - writer_info=self.writer_info, - warc_version=self.warc_version, - writer_subprefix=self.writer_subprefix, - gzip=self.gzip, - content_type=self.content_type, - min_part_size=self.min_part_size, - ) - tracker = ThroughputTracker() tracker.start() counter = 0 @@ -466,7 +448,7 @@ async def write_warc_records( if writer is None: # first file, with header and metadata records current_file_sequence = self.next_file_sequence(0) - writer, current_file_size = await self.start_file(current_file_sequence, **new_writer_kwargs) + writer, current_file_size = await self.start_file(current_file_sequence, s3_client) records_in_file = 0 elif records_in_file and self.max_file_size and \ current_file_size + len(item.data) > self.max_file_size: @@ -475,7 +457,7 @@ async def write_warc_records( await writer.close() writer = None current_file_sequence = self.next_file_sequence(current_file_sequence) - writer, current_file_size = await self.start_file(current_file_sequence, **new_writer_kwargs) + writer, current_file_size = await self.start_file(current_file_sequence, s3_client) records_in_file = 0 logger.info(f'Rotated to new WARC file sequence {current_file_sequence} due to size limit') @@ -524,15 +506,30 @@ def log_writer(self, counter: int, tracker: ThroughputTracker): stats['mb_per_sec'], ) - async def start_file(self, sequence: int, **new_writer_kwargs): - """Start a new WARC file with its warcinfo record (and the metadata records, if any). + async def start_file(self, sequence: int, s3_client=None): + """Start a new WARC file with its warcinfo record (and the metadata records, if any); if that fails, the + file is aborted. Returns the writer and the number of bytes written.""" - writer, file_size, warcinfo_id = await create_new_writer_with_header(sequence=sequence, **new_writer_kwargs) - if self.write_paths_as_metadata_records: - try: + writer, warc_file_name = new_warc_writer( + sequence, + self.prefix_path, + max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + min_part_size=self.min_part_size, + writer_subprefix=self.writer_subprefix, + gzip=self.gzip, + s3_client=s3_client, + ) + logger.debug('Starting WARC file %s', warc_file_name) + try: + await writer.start() + header, warcinfo_id = warcinfo_record_bytes(warc_file_name, self.writer_info, self.warc_version, self.gzip) + await writer.write(header) + file_size = len(header) + if self.write_paths_as_metadata_records: file_size += await self.write_metadata_records(writer, warcinfo_id=warcinfo_id) - except BaseException: - await writer.abort() - raise + except BaseException: + await writer.abort() + raise return writer, file_size diff --git a/cdx_toolkit/repackage/warc_utils.py b/cdx_toolkit/repackage/warc_utils.py index f39979b..7725deb 100644 --- a/cdx_toolkit/repackage/warc_utils.py +++ b/cdx_toolkit/repackage/warc_utils.py @@ -99,83 +99,35 @@ def existing_warc_files(prefix_path: str, writer_subprefix: Optional[str] = None ) -async def create_new_writer_with_header( +def new_warc_writer( sequence: int, output_path_prefix: str, max_attempts: int, base_backoff_seconds: float, min_part_size: int, - writer_info: Dict, - warc_version: str = '1.0', writer_subprefix: Optional[str] = None, gzip: bool = False, - content_type: Optional[str] = None, s3_client=None, -) -> Tuple[Union[S3ShardWriter, LocalFileWriter], int, str]: - """Create a new WARC writer (local or S3) and write the warcinfo record. +) -> Tuple[Union[S3ShardWriter, LocalFileWriter], str]: + """A new (not started) writer of a WARC file, local or S3, and the file's name (for the warcinfo record). - Returns the writer, the size of the warcinfo record and its WARC-Record-ID.""" + Both writers have the same life cycle: start(), write()..., then close(), which completes the file, or + abort(), which removes it (or aborts its upload); a writer that cannot close aborts itself.""" if is_s3_url(output_path_prefix): dest_bucket, dest_prefix = parse_s3_uri(output_path_prefix) + key = generate_warc_filename(dest_prefix, sequence=sequence, writer_subprefix=writer_subprefix, gzip=gzip) + writer = S3ShardWriter(s3_client, key, dest_bucket, min_part_size, max_attempts, base_backoff_seconds) + return writer, key.split('/')[-1] + + path = generate_warc_filename(output_path_prefix, sequence=sequence, writer_subprefix=writer_subprefix, gzip=gzip) + return LocalFileWriter(file_path=path), os.path.basename(path) # with backslashes on Windows + - warc_file_path = generate_warc_filename( - dest_prefix=dest_prefix, - sequence=sequence, - writer_subprefix=writer_subprefix, - gzip=gzip, - ) - - new_writer = S3ShardWriter( - s3_client, - warc_file_path, - dest_bucket, - content_type, - min_part_size, - max_attempts, - base_backoff_seconds, - ) - - else: - # local file system - warc_file_path = generate_warc_filename( - dest_prefix=output_path_prefix, - sequence=sequence, - writer_subprefix=writer_subprefix, - gzip=gzip, - ) - - new_writer = LocalFileWriter( - file_path=warc_file_path, - ) - - logger.debug('Initialzing new WARC writer for %s', warc_file_path) - - try: - return await _start_with_header(new_writer, warc_file_path, writer_info, warc_version, gzip) - except BaseException: - await new_writer.abort() - raise - - -async def _start_with_header(new_writer, warc_file_path: str, writer_info: Dict, warc_version: str, gzip: bool): - # Initialize writer - await new_writer.start() - - # Write WARC header; the `filename` field names this file - # an S3 key or a local path (with backslashes on Windows) - warc_file_name = warc_file_path.split('/')[-1] if isinstance(new_writer, S3ShardWriter) \ - else os.path.basename(warc_file_path) +def warcinfo_record_bytes(warc_file_name: str, writer_info: Dict, warc_version: str, gzip: bool) -> Tuple[bytes, str]: + """The warcinfo record of a WARC file (its `filename` field names the file) and its WARC-Record-ID (the + WARC-Warcinfo-ID of the file's other records).""" buffer = BytesIO() warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) - warcinfo = warc_writer.create_warcinfo_record( - filename=warc_file_name, - info=writer_info, - ) + warcinfo = warc_writer.create_warcinfo_record(filename=warc_file_name, info=writer_info) warc_writer.write_record(warcinfo) - header_data = buffer.getvalue() - await new_writer.write(header_data) - - # WARC-Warcinfo-ID indicates the WARC-Record-ID of the associated ‘warcinfo’ record - warcinfo_id = warcinfo.rec_headers.get('WARC-Record-ID') - - return new_writer, len(header_data), warcinfo_id + return buffer.getvalue(), warcinfo.rec_headers.get('WARC-Record-ID') diff --git a/tests/repackage/test_s3_writer.py b/tests/repackage/test_s3_writer.py index 97b369d..6637c39 100644 --- a/tests/repackage/test_s3_writer.py +++ b/tests/repackage/test_s3_writer.py @@ -73,7 +73,7 @@ async def abort_multipart_upload(self, **kwargs): def make_writer(s3): - writer = S3ShardWriter(s3, 'out.warc.gz', 'bucket', None, min_part_size=4, max_attempts=2, + writer = S3ShardWriter(s3, 'out.warc.gz', 'bucket', min_part_size=4, max_attempts=2, base_backoff_seconds=0) writer.upload_id = 'upload-id' return writer From a2685875885058b06e9c7ece9272f0c4e82696ad Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 00:07:19 +0200 Subject: [PATCH 36/91] refactor(index): DuckDB loads httpfs/aws and AWS credentials only for remote files --- cdx_toolkit/indexes/duckdb.py | 41 ++++++++++++++++++++++------------- tests/unit/test_duckdb.py | 30 ++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/cdx_toolkit/indexes/duckdb.py b/cdx_toolkit/indexes/duckdb.py index 65bbf5a..4798f57 100644 --- a/cdx_toolkit/indexes/duckdb.py +++ b/cdx_toolkit/indexes/duckdb.py @@ -30,6 +30,29 @@ def build_from_clause(index_path: str, crawls: Optional[List[str]]) -> str: return f"read_parquet([{', '.join(globs)}], hive_partitioning=true)" +def _configure_httpfs(con, s3: bool, region_name: Optional[str] = None) -> None: + """Load the httpfs extension (http(s):// and s3://); for s3://, also find AWS credentials and apply the S3 + settings.""" + con.execute('INSTALL httpfs; LOAD httpfs;') + settings = ['SET http_timeout=120;'] # 2 minutes for large parquet partitions + if s3: + region = aws_region(region_name) + try: + # find AWS credentials like boto3 does (env, ~/.aws profiles, SSO, instance roles) + con.execute('INSTALL aws; LOAD aws;') + con.execute(f"CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION '{region}');") + except Exception as e: # pragma: no cover - no credentials or aws extension unavailable + logger.debug('duckdb AWS credential chain not used: %r', e) + # region, retries and URL style: Common Crawl's recommended S3 settings, unless configured by + # the user (see cdx_toolkit.aws) + settings = duckdb_settings(region_name) + settings + for stmt in settings: + try: + con.execute(stmt) + except Exception as e: # pragma: no cover - setting unsupported on this duckdb + logger.debug('duckdb setting skipped: %s (%r)', stmt, e) + + def iter_duckdb_rows(query: str, region_name: Optional[str] = None) -> Iterator[dict]: """Run the query and yield the result rows as dicts (column name -> value). @@ -44,23 +67,11 @@ def iter_duckdb_rows(query: str, region_name: Optional[str] = None) -> Iterator[ con = duckdb.connect() try: - con.execute('INSTALL httpfs; LOAD httpfs;') # CC timestamps are UTC (timestamp literals in queries, timestamps in the results) con.execute("SET TimeZone='UTC';") - region = aws_region(region_name) - try: - # find AWS credentials like boto3 does (env, ~/.aws profiles, SSO, instance roles) - con.execute('INSTALL aws; LOAD aws;') - con.execute(f"CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION '{region}');") - except Exception as e: # pragma: no cover - no credentials or aws extension unavailable - logger.debug('duckdb AWS credential chain not used: %r', e) - # region, retries and URL style: Common Crawl's recommended S3 settings, unless configured by - # the user (see cdx_toolkit.aws); a timeout of 2 minutes for large parquet partitions - for stmt in duckdb_settings(region_name) + ['SET http_timeout=120;']: - try: - con.execute(stmt) - except Exception as e: # pragma: no cover - setting unsupported on this duckdb - logger.debug('duckdb setting skipped: %s (%r)', stmt, e) + # remote files only as needed: a local query needs no extensions (nor AWS credentials) + if 's3://' in query or 'http://' in query or 'https://' in query: + _configure_httpfs(con, s3='s3://' in query, region_name=region_name) if 'hf://buckets/' in query: # DuckDB cannot read Hugging Face Storage Buckets itself; use their fsspec filesystem. from huggingface_hub import HfFileSystem diff --git a/tests/unit/test_duckdb.py b/tests/unit/test_duckdb.py index fa31d70..8256d87 100644 --- a/tests/unit/test_duckdb.py +++ b/tests/unit/test_duckdb.py @@ -39,6 +39,34 @@ def test_without_duckdb(): list(iter_duckdb_rows('SELECT 1')) +def run_with_fake_connection(query): + """The rows of the query, the executed statements and the connection.""" + con = MagicMock() + con.execute.return_value.description = [('n',)] + con.execute.return_value.fetchmany.side_effect = [[(1,)], []] + fake_duckdb = MagicMock(connect=MagicMock(return_value=con)) + with patch.object(duckdb_source, '_HAS_DUCKDB', True), \ + patch.object(duckdb_source, 'duckdb', fake_duckdb, create=True): + rows = list(iter_duckdb_rows(query)) + return rows, [c.args[0] for c in con.execute.call_args_list], con + + +@pytest.mark.parametrize('query, httpfs, s3', [ + ("SELECT 1 AS n FROM read_parquet('/data/index/*.parquet')", False, False), + ("SELECT 1 AS n FROM read_parquet('https://data.commoncrawl.org/x.parquet')", True, False), + ("SELECT 1 AS n FROM read_parquet('s3://commoncrawl/x.parquet')", True, True), +]) +def test_extensions_and_credentials_only_when_needed(monkeypatch, query, httpfs, s3): + monkeypatch.setattr('cdx_toolkit.aws.user_aws_config', lambda: {}) + rows, statements, _ = run_with_fake_connection(query) + assert rows == [{'n': 1}] + assert ('INSTALL httpfs; LOAD httpfs;' in statements) == httpfs + assert ('SET http_timeout=120;' in statements) == httpfs + assert ('INSTALL aws; LOAD aws;' in statements) == s3 + assert any(stmt.startswith('SET s3_region=') for stmt in statements) == s3 + assert statements[-1] == query + + def test_connection_setup_and_hf_filesystem(monkeypatch): for name in ('AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_MAX_ATTEMPTS', 'CDXT_AWS_DEFAULTS'): monkeypatch.delenv(name, raising=False) @@ -50,7 +78,7 @@ def test_connection_setup_and_hf_filesystem(monkeypatch): hf_fs = MagicMock() fake_hf = MagicMock(HfFileSystem=MagicMock(return_value=hf_fs)) - query = "SELECT count(*) AS n FROM read_parquet('hf://buckets/x/y/*.parquet')" + query = "SELECT count(*) AS n FROM read_parquet(['hf://buckets/x/y/*.parquet', 's3://commoncrawl/x.parquet'])" with patch.object(duckdb_source, '_HAS_DUCKDB', True), \ patch.object(duckdb_source, 'duckdb', fake_duckdb, create=True), \ patch.dict('sys.modules', {'huggingface_hub': fake_hf}): From 26727d7aca32546a5a478853024acbcd48af71c3 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 00:07:58 +0200 Subject: [PATCH 37/91] test: check AWS access when a live test runs, not when the tests are collected --- tests/conftest.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0d7eaa3..a731d07 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -132,11 +132,16 @@ def requires_live_aws(func): return pytest.mark.skip(reason='needs AWS; set CDXT_LIVE_TESTS=1 to run it')(func) if not _HAS_BOTOCORE or not _HAS_FSSPEC: return pytest.mark.skip(reason='S3 dependencies are not installed; install cdx_toolkit[s3]')(func) - return pytest.mark.skipif(DISABLE_S3_TESTS, reason='AWS S3 access is disabled via environment variable.')( - pytest.mark.skipif( - not check_aws_s3_access(), reason='AWS S3 access not available (no credentials or permissions)' - )(func) - ) + return pytest.mark.usefixtures('live_aws_s3_access')(func) + + +@pytest.fixture +def live_aws_s3_access(): + """Skip the test without AWS S3 access; checked when a test needs it (not when the tests are collected).""" + if DISABLE_S3_TESTS: + pytest.skip('AWS S3 access is disabled via environment variable.') + if not check_aws_s3_access(): + pytest.skip('AWS S3 access not available (no credentials or permissions)') def requires_aws_s3(func): @@ -200,11 +205,16 @@ def requires_aws_athena(func): True, reason='S3 dependencies are not installed; install cdx_toolkit[s3] to enable Athena tests.' )(func) - return pytest.mark.skipif(DISABLE_ATHENA_TESTS, reason='AWS Athena access is disabled via environment variable.')( - pytest.mark.skipif( - not check_aws_athena_access(), reason='AWS Athena access not available (no credentials or permissions)' - )(func) - ) + return pytest.mark.usefixtures('live_aws_athena_access')(func) + + +@pytest.fixture +def live_aws_athena_access(): + """Skip the test without AWS Athena access; checked when a test needs it (not when the tests are collected).""" + if DISABLE_ATHENA_TESTS: + pytest.skip('AWS Athena access is disabled via environment variable.') + if not check_aws_athena_access(): + pytest.skip('AWS Athena access not available (no credentials or permissions)') def requires_duckdb(func): From ef6d7dab9a73449530a2205829930ade87f90c55 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 00:09:02 +0200 Subject: [PATCH 38/91] docs: changelog: cdxt repackage does not overwrite an existing series --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 031af69..6d0b545 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,8 @@ + `--output-format cdx` / `--index-format cdx` as another name for `cdxj`; clear error for globs over http(s) + Parquet output for `cdxt index` and Parquet input for `cdxt repackage` (`cdx_toolkit[parquet]`) + `cdxt repackage`: default `--prefix REPACKAGE` (was `TEST`), no WARC file is written without captures, - a full URL in `warc_filename`/`filename` is used as it is (the `warc_url` column is gone) + a full URL in `warc_filename`/`filename` is used as it is (the `warc_url` column is gone); stops (exit status 2) + if files of the output series exist already, instead of overwriting them + fixed `--to` with a day (e.g. `--to 20240410`), which was padded to the end of the month, and February in leap years + removed the deprecated `cdx_iter` and `cdx_size` scripts + S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual From 20a1fd442f75e6ea541c43f2245d92607e44757f Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 00:14:30 +0200 Subject: [PATCH 39/91] fix(repackage): --wb also rejects existing WARC files of the prefix (same rule for both writers) --- CHANGELOG.md | 3 ++- README.md | 9 ++++++--- cdx_toolkit/cli/repackage.py | 25 +++++++++++++++---------- cdx_toolkit/repackage/warc_utils.py | 5 +++-- tests/repackage/test_warc_utils.py | 6 +++--- tests/unit/test_cli.py | 24 ++++++++++++++++++++++++ 6 files changed, 53 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d0b545..2f4aa03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,8 @@ + Parquet output for `cdxt index` and Parquet input for `cdxt repackage` (`cdx_toolkit[parquet]`) + `cdxt repackage`: default `--prefix REPACKAGE` (was `TEST`), no WARC file is written without captures, a full URL in `warc_filename`/`filename` is used as it is (the `warc_url` column is gone); stops (exit status 2) - if files of the output series exist already, instead of overwriting them + if WARC files with the same prefix exist already, instead of overwriting them (also with `--wb`, which + numbered its files after the existing ones before) + fixed `--to` with a day (e.g. `--to 20240410`), which was padded to the end of the month, and February in leap years + removed the deprecated `cdx_iter` and `cdx_size` scripts + S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual diff --git a/README.md b/README.md index 3af465a..23f64f3 100644 --- a/README.md +++ b/README.md @@ -187,9 +187,7 @@ written to `[-]-001.warc.gz`, `-002`, ..., where the prefix can be a local path, `s3://...` or `hf://buckets/...` (default: `REPACKAGE` in the current directory, e.g. `--prefix out/CC-HOMEPAGES` writes `out/CC-HOMEPAGES-001.warc.gz`). No file is written when there are no -captures. If files of this series exist already (e.g. from an earlier run), -`cdxt repackage` stops before fetching (exit status 2): use another -`--prefix` or `--subprefix`. The captures are +captures. The captures are sorted by WARC file and offset first (for faster reads); `--no-sort` keeps the input order and streams large inputs without keeping them in memory. @@ -210,6 +208,11 @@ replayed from the wayback machine given with `--wb` (`ia` is a shortcut for `https://web.archive.org/web`) and written one by one to `[-]-000000.extracted.warc.gz`, ... +In both cases, if WARC files with the same prefix and subprefix exist already +(e.g. from an earlier run), `cdxt repackage` stops before fetching (exit +status 2) instead of overwriting them or adding to them: use another +`--prefix` or `--subprefix`. + Use `--size` (e.g. `100MB` or `1GiB`) to set the target size of each WARC file, `--creator`, `--operator`, `--is-part-of` and `--description` to fill in the `warcinfo` record, and `--add-metadata-record FILE` to add a file diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index 507978f..99a80da 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -25,11 +25,12 @@ Captures with a WARC location (filename, offset, length; e.g. Common Crawl captures) are copied byte-for-byte from --warc-download-prefix (https://, s3:// or hf://buckets/...) into -[-]-001.warc.gz, -002, ... (local, s3:// or hf://buckets/...); files of this -series must not exist yet. +[-]-001.warc.gz, -002, ... (local, s3:// or hf://buckets/...). Captures without a WARC location (e.g. Internet Archive captures) are replayed from a wayback machine with --wb and written to [-]-000000.extracted.warc.gz, ... + +WARC files with the same prefix and subprefix (of either kind) must not exist yet. ''' EXAMPLES = '''\ @@ -149,6 +150,7 @@ def run(args: Namespace, cmdline: str): import_pyarrow() except RuntimeError as e: raise UsageError(str(e)) from None + check_output_is_new(args.prefix, args.subprefix) records = read_records(args.input, index_format) info = build_warcinfo(args, cmdline) start_time = time.time() @@ -167,22 +169,25 @@ def run(args: Namespace, cmdline: str): ) -def repackage_by_warc_location(args: Namespace, records, info: dict): - """Copy the WARC records byte-for-byte with the async WARC fetcher (optionally multi-process).""" - import fsspec - - from cdx_toolkit.repackage.hf_utils import is_hf_url - from cdx_toolkit.repackage.records_source import MissingWarcLocation, RecordsSource +def check_output_is_new(prefix: str, subprefix: Optional[str]) -> None: + """Every run numbers its files from the start: never overwrite (or mix with) the files of an earlier run, with + or without --wb.""" from cdx_toolkit.repackage.warc_utils import existing_warc_files - # every run numbers its files from 001: never overwrite (or mix with) the files of an earlier run - existing = existing_warc_files(args.prefix, args.subprefix) + existing = existing_warc_files(prefix, subprefix) if existing: raise UsageError( f'{len(existing)} WARC file(s) with this --prefix/--subprefix exist already, e.g. {existing[0]}; ' 'use another prefix or subprefix, or remove them' ) + +def repackage_by_warc_location(args: Namespace, records, info: dict): + """Copy the WARC records byte-for-byte with the async WARC fetcher (optionally multi-process).""" + import fsspec + + from cdx_toolkit.repackage.hf_utils import is_hf_url + from cdx_toolkit.repackage.records_source import MissingWarcLocation, RecordsSource try: source = RecordsSource(records, args.warc_download_prefix, sort=not args.no_sort) diff --git a/cdx_toolkit/repackage/warc_utils.py b/cdx_toolkit/repackage/warc_utils.py index 7725deb..486a32c 100644 --- a/cdx_toolkit/repackage/warc_utils.py +++ b/cdx_toolkit/repackage/warc_utils.py @@ -87,12 +87,13 @@ def generate_warc_filename( def existing_warc_files(prefix_path: str, writer_subprefix: Optional[str] = None, gzip: bool = True) -> List[str]: - """The files of the series [-]-NNN.warc[.gz] that already exist (local, s3://, hf://, ...).""" + """The files of the series [-]-NNN.warc[.gz] (async WARC fetcher) or + [-]-NNNNNN.extracted.warc[.gz] (--wb) that already exist (local, s3://, hf://, ...).""" fs, fs_prefix = fsspec.url_to_fs(prefix_path) name = re.escape(posixpath.basename(fs_prefix)) + '-' if writer_subprefix is not None: name += re.escape(writer_subprefix) + '-' - name += r'\d{3,}\.warc' + (r'\.gz' if gzip else '') + name += r'\d{3,}(\.extracted)?\.warc' + (r'\.gz' if gzip else '') return sorted( fs.unstrip_protocol(path) for path in fs.glob(glob.escape(fs_prefix) + '-*') if re.fullmatch(name, posixpath.basename(path)) diff --git a/tests/repackage/test_warc_utils.py b/tests/repackage/test_warc_utils.py index 3a21c94..cf77c4c 100644 --- a/tests/repackage/test_warc_utils.py +++ b/tests/repackage/test_warc_utils.py @@ -18,13 +18,13 @@ def test_get_metadata_record_from_path(): @pytest.mark.parametrize('subprefix, expected', [ - (None, ['out-001.warc.gz', 'out-1000.warc.gz']), + (None, ['out-000000.extracted.warc.gz', 'out-001.warc.gz', 'out-1000.warc.gz']), ('SUB', ['out-SUB-002.warc.gz']), ('OTHER', []), ]) def test_existing_warc_files(tmp_path, subprefix, expected): - for name in ['out-001.warc.gz', 'out-1000.warc.gz', 'out-SUB-002.warc.gz', 'out-x.warc.gz', 'out-001.warc', - 'outer-001.warc.gz', 'other-001.warc.gz']: + for name in ['out-001.warc.gz', 'out-1000.warc.gz', 'out-000000.extracted.warc.gz', 'out-x.extracted.warc.gz', + 'out-SUB-002.warc.gz', 'out-x.warc.gz', 'out-001.warc', 'outer-001.warc.gz', 'other-001.warc.gz']: (tmp_path / name).write_bytes(b'') found = existing_warc_files(str(tmp_path / 'out'), subprefix) assert [f.rsplit('/', 1)[-1] for f in found] == expected diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 70758b9..612a326 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -79,6 +79,30 @@ def test_repackage_fetches_by_warc_location(tmp_path): assert [r.rec_type for r in records] == ['warcinfo', 'response'] +@pytest.mark.parametrize('existing, wb', [ + ('out-000000.extracted.warc.gz', True), + ('out-001.warc.gz', True), + ('out-000000.extracted.warc.gz', False), +]) +def test_repackage_rejects_existing_warc_files_with_and_without_wb(tmp_path, capsys, existing, wb): + """The same rule for both writers: no overwriting, no adding to the files of an earlier run.""" + (tmp_path / existing).write_bytes(b'earlier run') + captures = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE if wb else CC_CAPTURE]) + wb_args = ['--wb', 'ia'] if wb else [] + + with patch.object(cdx_toolkit.cli.repackage, 'fetch_wb_warc') as fetch, \ + patch('cdx_toolkit.repackage.data_classes.myrequests_get') as get, \ + pytest.raises(SystemExit) as exc: + main(['repackage', captures, '--prefix', str(tmp_path / 'out')] + wb_args) + + assert exc.value.code == 2 + assert 'exist already' in capsys.readouterr().err + fetch.assert_not_called() + get.assert_not_called() + assert sorted(p.name for p in tmp_path.glob('out-*')) == [existing] + assert (tmp_path / existing).read_bytes() == b'earlier run' + + def test_repackage_without_warc_location_or_wb(tmp_path, capsys): captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE, IA_CAPTURE]) From 6962d20a727a5721536ef5480026d37545394249 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 12:44:15 +0200 Subject: [PATCH 40/91] refactor(index): move the backends to cdx_toolkit.indexes.backends (one module per backend) DuckDbBackend and AthenaBackend live with their engine helpers (backends/duckdb.py, backends/athena.py), backends/sql.py has the common SqlBackend, and sql_query.py is renamed to backends/sql_builder.py, which now also has the predicate helpers of the guided queries (timestamps, languages, --url SURT keys). --- cdx_toolkit/cli/index.py | 2 +- cdx_toolkit/indexes/backends/__init__.py | 4 + cdx_toolkit/indexes/{ => backends}/athena.py | 55 ++++++- cdx_toolkit/indexes/{ => backends}/cdx_api.py | 4 +- .../indexes/{ => backends}/cdx_files.py | 6 +- cdx_toolkit/indexes/{ => backends}/duckdb.py | 33 +++- cdx_toolkit/indexes/{ => backends}/sql.py | 142 +----------------- .../{sql_query.py => backends/sql_builder.py} | 60 +++++++- cdx_toolkit/indexes/resolve.py | 8 +- tests/unit/test_athena.py | 15 +- tests/unit/test_cdx_files.py | 2 +- tests/unit/test_confirm_cost.py | 12 +- tests/unit/test_duckdb.py | 4 +- tests/unit/test_indexes.py | 4 +- ...{test_sql_query.py => test_sql_builder.py} | 6 +- tests/unit/test_sql_indexes.py | 17 ++- tests/unit/test_usage_errors.py | 8 +- 17 files changed, 195 insertions(+), 187 deletions(-) create mode 100644 cdx_toolkit/indexes/backends/__init__.py rename cdx_toolkit/indexes/{ => backends}/athena.py (68%) rename cdx_toolkit/indexes/{ => backends}/cdx_api.py (98%) rename cdx_toolkit/indexes/{ => backends}/cdx_files.py (98%) rename cdx_toolkit/indexes/{ => backends}/duckdb.py (73%) rename cdx_toolkit/indexes/{ => backends}/sql.py (54%) rename cdx_toolkit/indexes/{sql_query.py => backends/sql_builder.py} (74%) rename tests/unit/{test_sql_query.py => test_sql_builder.py} (92%) diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index 34cf1e7..32cc225 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -209,7 +209,7 @@ def get_output_format(args: Namespace) -> str: def writes_per_file(args: Namespace, backend) -> bool: """CDX files with an output path without file extension: one output per CDX file.""" - from cdx_toolkit.indexes.cdx_files import CdxFilesBackend + from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend return ( isinstance(backend, CdxFilesBackend) diff --git a/cdx_toolkit/indexes/backends/__init__.py b/cdx_toolkit/indexes/backends/__init__.py new file mode 100644 index 0000000..3835f43 --- /dev/null +++ b/cdx_toolkit/indexes/backends/__init__.py @@ -0,0 +1,4 @@ +''' +The index backends, one module each: the CDX API (cdx_api), CDX files (cdx_files), and the columnar URL index +queried with SQL (sql: common logic, sql_builder: the SQL text) via DuckDB (duckdb) or AWS Athena (athena). +''' diff --git a/cdx_toolkit/indexes/athena.py b/cdx_toolkit/indexes/backends/athena.py similarity index 68% rename from cdx_toolkit/indexes/athena.py rename to cdx_toolkit/indexes/backends/athena.py index 27b5097..9105d78 100644 --- a/cdx_toolkit/indexes/athena.py +++ b/cdx_toolkit/indexes/backends/athena.py @@ -1,12 +1,17 @@ ''' -Running queries with AWS Athena (boto3). +Index backend for the columnar URL index queried with AWS Athena (boto3). ''' import logging +import re import time -from typing import Iterable, Iterator, Optional +from typing import Iterable, Iterator, Optional, Tuple +from urllib.parse import parse_qs, urlparse from cdx_toolkit.aws import botocore_config +from ..base import UsageError +from .sql import SqlBackend + logger = logging.getLogger(__name__) @@ -129,3 +134,49 @@ def iter_result_rows(client, query_execution_id: str) -> Iterable[dict]: for row in rows: row_data = [cell.get('VarCharValue', None) for cell in row['Data']] yield dict(zip(column_names, row_data)) + + +class AthenaBackend(SqlBackend): + """The columnar index queried with AWS Athena. + + Location: athena://athena..amazonaws.com[:443]/?S3OutputLocation=s3://...[&table=ccindex] + """ + + name = 'Athena' + engine = 'athena' + url_file_hint = ( + ' (after creating an Athena table `target_urls` from the URL file on S3):\n\n' + ' SELECT idx.url, idx.warc_filename, idx.warc_record_offset, idx.warc_record_length\n' + ' FROM ccindex AS idx\n' + ' JOIN target_urls AS target ON idx.url = target.url\n' + " WHERE idx.crawl = 'CC-MAIN-2024-33' AND idx.subset = 'warc'" + ) + + def __init__(self, location: str, assume_yes: bool = False): + super().__init__(location, assume_yes) + self.region, self.database, self.table, self.s3_output_location = parse_athena_location(location) + + def from_clause(self, crawls): + return self.table, False + + def iter_sql_rows(self, sql): + return iter_athena_rows(sql, self.database, self.s3_output_location, region_name=self.region) + + +def parse_athena_location(location: str) -> Tuple[str, str, str, str]: + """athena://athena.us-east-1.amazonaws.com:443/ccindex?S3OutputLocation=s3://bucket/path/ + -> (region, database, table, S3 output location)""" + parsed = urlparse(location) + match = re.fullmatch(r'athena\.([a-z0-9-]+)\.amazonaws\.com', parsed.hostname or '') + if parsed.scheme != 'athena' or not match: + raise UsageError( + f'invalid Athena location {location!r}, expected ' + 'athena://athena..amazonaws.com/?S3OutputLocation=s3:////' + ) + params = parse_qs(parsed.query) + s3_output_location = params.get('S3OutputLocation', [None])[0] + if not s3_output_location or not s3_output_location.startswith('s3://'): + raise UsageError(f'the Athena location {location!r} needs a S3OutputLocation=s3://... parameter') + database = parsed.path.strip('/') or 'ccindex' + table = params.get('table', ['ccindex'])[0] + return match.group(1), database, table, s3_output_location diff --git a/cdx_toolkit/indexes/cdx_api.py b/cdx_toolkit/indexes/backends/cdx_api.py similarity index 98% rename from cdx_toolkit/indexes/cdx_api.py rename to cdx_toolkit/indexes/backends/cdx_api.py index 84db22a..a6f453c 100644 --- a/cdx_toolkit/indexes/cdx_api.py +++ b/cdx_toolkit/indexes/backends/cdx_api.py @@ -4,8 +4,8 @@ import cdx_toolkit from cdx_toolkit.commoncrawl import normalize_crawl -from .base import IndexBackend, UsageError -from .query import IndexQuery +from ..base import IndexBackend, UsageError +from ..query import IndexQuery LOGGER = logging.getLogger(__name__) diff --git a/cdx_toolkit/indexes/cdx_files.py b/cdx_toolkit/indexes/backends/cdx_files.py similarity index 98% rename from cdx_toolkit/indexes/cdx_files.py rename to cdx_toolkit/indexes/backends/cdx_files.py index 981d197..d354be1 100644 --- a/cdx_toolkit/indexes/cdx_files.py +++ b/cdx_toolkit/indexes/backends/cdx_files.py @@ -14,12 +14,12 @@ from cdx_toolkit.record_io import check_glob, format_from_path, is_glob, open_text, parse_cdxj_line, read_records from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up -from .base import IndexBackend, UsageError -from .patterns import ( # noqa: F401 (url_pattern_to_surts, to_surt) +from ..base import IndexBackend, UsageError +from ..patterns import ( # noqa: F401 (url_pattern_to_surts, to_surt) domain_entry_to_surts, domain_to_surts, hostname_to_surts, normalize_host, to_surt, url_hostname, url_pattern_to_surts, ) -from .query import IndexQuery +from ..query import IndexQuery LOGGER = logging.getLogger(__name__) diff --git a/cdx_toolkit/indexes/duckdb.py b/cdx_toolkit/indexes/backends/duckdb.py similarity index 73% rename from cdx_toolkit/indexes/duckdb.py rename to cdx_toolkit/indexes/backends/duckdb.py index 4798f57..447b8a1 100644 --- a/cdx_toolkit/indexes/duckdb.py +++ b/cdx_toolkit/indexes/backends/duckdb.py @@ -1,11 +1,14 @@ ''' -Running queries with DuckDB, e.g. over the columnar URL index on S3 or on Hugging Face. +Index backend for the columnar URL index queried with DuckDB, e.g. on S3, on Hugging Face or local. ''' import logging from typing import Iterator, List, Optional from cdx_toolkit.aws import aws_region, duckdb_settings +from ..base import UsageError +from .sql import SqlBackend + logger = logging.getLogger(__name__) @@ -89,3 +92,31 @@ def iter_duckdb_rows(query: str, region_name: Optional[str] = None) -> Iterator[ yield dict(zip(col_names, row)) finally: con.close() + + +class DuckDbBackend(SqlBackend): + """The columnar index queried with DuckDB: a hive-partitioned directory or parquet files (local, s3, hf).""" + + name = 'DuckDB' + engine = 'duckdb' + url_file_hint = ( + ':\n\n' + ' SELECT idx.url, idx.warc_filename, idx.warc_record_offset, idx.warc_record_length\n' + " FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-33/subset=warc/" + "*.parquet') AS idx\n" + " JOIN read_csv('urls.csv') AS target ON idx.url = target.url" + ) + + def __init__(self, location: str, assume_yes: bool = False): + if "'" in location: + raise UsageError(f'invalid index location {location!r}') + super().__init__(location, assume_yes) + + def from_clause(self, crawls): + if self.location.endswith('.parquet'): # file(s), possibly a glob + return f"read_parquet('{self.location}', hive_partitioning=true)", False + # hive-partitioned index directory: prune the crawls in the glob + return build_from_clause(self.location, crawls), True + + def iter_sql_rows(self, sql): + return iter_duckdb_rows(sql) diff --git a/cdx_toolkit/indexes/sql.py b/cdx_toolkit/indexes/backends/sql.py similarity index 54% rename from cdx_toolkit/indexes/sql.py rename to cdx_toolkit/indexes/backends/sql.py index 1cc5267..e5b57b8 100644 --- a/cdx_toolkit/indexes/sql.py +++ b/cdx_toolkit/indexes/backends/sql.py @@ -1,19 +1,15 @@ ''' -Index backends for the columnar URL index (parquet), queried with SQL via AWS Athena or DuckDB. +The common logic of the backends for the columnar URL index (parquet), queried with SQL (see duckdb.py, athena.py). ''' import datetime import logging import re import sys from typing import Iterator, List, Mapping, Optional, Tuple -from urllib.parse import parse_qs, urlparse -from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up - -from .base import IndexBackend, UsageError -from .patterns import url_pattern_to_surts -from .query import IndexQuery, field_flag -from .sql_query import build_sql, resolve_crawl_names +from ..base import IndexBackend, UsageError +from ..query import IndexQuery, field_flag +from .sql_builder import build_sql, query_conditions, resolve_crawl_names, url_surtkeys LOGGER = logging.getLogger(__name__) @@ -98,56 +94,6 @@ def normalize_row(row: Mapping) -> dict: return record -def sql_timestamp(timestamp: str, end: bool = False) -> str: - """CDX timestamp (e.g. 2008, 200906302359) -> SQL TIMESTAMP literal of its first second, or with `end`, of the - second after its last second (an exclusive end, which includes the fractions of the last second).""" - if not re.fullmatch(r'\d{4,14}', timestamp): - raise UsageError(f'invalid timestamp {timestamp!r}, use e.g. 2008, 20080328 or 200906302359') - padded = pad_timestamp_up(timestamp) if end else pad_timestamp(timestamp) - try: - moment = datetime.datetime.strptime(padded, '%Y%m%d%H%M%S') - if end: - moment += datetime.timedelta(seconds=1) - except (ValueError, OverflowError): - raise UsageError(f'invalid timestamp {timestamp!r}') from None - return f"TIMESTAMP '{moment:%Y-%m-%d %H:%M:%S}'" - - -def check_language(code: str) -> str: - """An ISO 639-3 language code (three letters, e.g. eng or deu), in lowercase.""" - language = code.strip().lower() - if not re.fullmatch(r'[a-z]{3}', language): - raise UsageError(f'invalid language {code!r}, use ISO 639-3 codes (three letters), e.g. eng or deu') - return language - - -def url_surtkeys(query: IndexQuery) -> Tuple[List[str], List[str]]: - """--url patterns -> (url_surtkey prefixes, exact url_surtkey values).""" - prefixes, exact = [], [] - for pattern in query.urls: - p, e = url_pattern_to_surts(pattern) - prefixes += p - exact += e - return prefixes, exact - - -def query_conditions(query: IndexQuery) -> List[str]: - """WHERE predicates for --from, --to and --language.""" - conditions = [] - if query.from_ts: - conditions.append(f'fetch_time >= {sql_timestamp(query.from_ts)}') - if query.to: - conditions.append(f'fetch_time < {sql_timestamp(query.to, end=True)}') - if query.languages: - # primary content language: the first of the comma-separated content_languages - predicates = [] - for code in query.languages: - language = check_language(code) - predicates += [f"content_languages = '{language}'", f"content_languages LIKE '{language},%'"] - conditions.append('(' + ' OR '.join(predicates) + ')') - return conditions - - class SqlBackend(IndexBackend): """Common logic of the SQL backends: guided query building, cost guard, row normalization.""" @@ -225,83 +171,3 @@ def explain(self, query: IndexQuery) -> str: sql, n_crawls = self.prepare(query) scope = f'{n_crawls} crawl(s)' if n_crawls else 'all crawls (or not verifiable)' return f'-- {self.name} query on {self.location}, restricted to {scope}\n{sql.strip()}\n' - - -class DuckDbBackend(SqlBackend): - """The columnar index queried with DuckDB: a hive-partitioned directory or parquet files (local, s3, hf).""" - - name = 'DuckDB' - engine = 'duckdb' - url_file_hint = ( - ':\n\n' - ' SELECT idx.url, idx.warc_filename, idx.warc_record_offset, idx.warc_record_length\n' - " FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-33/subset=warc/" - "*.parquet') AS idx\n" - " JOIN read_csv('urls.csv') AS target ON idx.url = target.url" - ) - - def __init__(self, location: str, assume_yes: bool = False): - if "'" in location: - raise UsageError(f'invalid index location {location!r}') - super().__init__(location, assume_yes) - - def from_clause(self, crawls): - from .duckdb import build_from_clause - - if self.location.endswith('.parquet'): # file(s), possibly a glob - return f"read_parquet('{self.location}', hive_partitioning=true)", False - # hive-partitioned index directory: prune the crawls in the glob - return build_from_clause(self.location, crawls), True - - def iter_sql_rows(self, sql): - from .duckdb import iter_duckdb_rows - - return iter_duckdb_rows(sql) - - -class AthenaBackend(SqlBackend): - """The columnar index queried with AWS Athena. - - Location: athena://athena..amazonaws.com[:443]/?S3OutputLocation=s3://...[&table=ccindex] - """ - - name = 'Athena' - engine = 'athena' - url_file_hint = ( - ' (after creating an Athena table `target_urls` from the URL file on S3):\n\n' - ' SELECT idx.url, idx.warc_filename, idx.warc_record_offset, idx.warc_record_length\n' - ' FROM ccindex AS idx\n' - ' JOIN target_urls AS target ON idx.url = target.url\n' - " WHERE idx.crawl = 'CC-MAIN-2024-33' AND idx.subset = 'warc'" - ) - - def __init__(self, location: str, assume_yes: bool = False): - super().__init__(location, assume_yes) - self.region, self.database, self.table, self.s3_output_location = parse_athena_location(location) - - def from_clause(self, crawls): - return self.table, False - - def iter_sql_rows(self, sql): - from .athena import iter_athena_rows - - return iter_athena_rows(sql, self.database, self.s3_output_location, region_name=self.region) - - -def parse_athena_location(location: str) -> Tuple[str, str, str, str]: - """athena://athena.us-east-1.amazonaws.com:443/ccindex?S3OutputLocation=s3://bucket/path/ - -> (region, database, table, S3 output location)""" - parsed = urlparse(location) - match = re.fullmatch(r'athena\.([a-z0-9-]+)\.amazonaws\.com', parsed.hostname or '') - if parsed.scheme != 'athena' or not match: - raise UsageError( - f'invalid Athena location {location!r}, expected ' - 'athena://athena..amazonaws.com/?S3OutputLocation=s3:////' - ) - params = parse_qs(parsed.query) - s3_output_location = params.get('S3OutputLocation', [None])[0] - if not s3_output_location or not s3_output_location.startswith('s3://'): - raise UsageError(f'the Athena location {location!r} needs a S3OutputLocation=s3://... parameter') - database = parsed.path.strip('/') or 'ccindex' - table = params.get('table', ['ccindex'])[0] - return match.group(1), database, table, s3_output_location diff --git a/cdx_toolkit/indexes/sql_query.py b/cdx_toolkit/indexes/backends/sql_builder.py similarity index 74% rename from cdx_toolkit/indexes/sql_query.py rename to cdx_toolkit/indexes/backends/sql_builder.py index 1a2e039..80bac8a 100644 --- a/cdx_toolkit/indexes/sql_query.py +++ b/cdx_toolkit/indexes/backends/sql_builder.py @@ -1,13 +1,17 @@ ''' -SQL queries for the columnar URL index (Athena, DuckDB): escaping, the WHERE clause of the guided queries, crawls. +SQL text for the columnar URL index (Athena, DuckDB): escaping, the WHERE clause of the guided queries, crawls. ''' +import datetime import logging import re -from typing import List, Optional +from typing import List, Optional, Tuple from cdx_toolkit.commoncrawl import normalize_crawl, get_cc_endpoints, match_cc_crawls +from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up -from .patterns import domain_to_surts, normalize_host +from ..base import UsageError +from ..patterns import domain_to_surts, normalize_host, url_pattern_to_surts +from ..query import IndexQuery logger = logging.getLogger(__name__) @@ -174,3 +178,53 @@ def resolve_crawl_names(crawl_arg) -> List[str]: raw_index_list = get_cc_endpoints(CC_INDEX_MIRROR) matched = match_cc_crawls(crawls, raw_index_list) return [endpoint_to_crawl_name(ep) for ep in matched] + + +def sql_timestamp(timestamp: str, end: bool = False) -> str: + """CDX timestamp (e.g. 2008, 200906302359) -> SQL TIMESTAMP literal of its first second, or with `end`, of the + second after its last second (an exclusive end, which includes the fractions of the last second).""" + if not re.fullmatch(r'\d{4,14}', timestamp): + raise UsageError(f'invalid timestamp {timestamp!r}, use e.g. 2008, 20080328 or 200906302359') + padded = pad_timestamp_up(timestamp) if end else pad_timestamp(timestamp) + try: + moment = datetime.datetime.strptime(padded, '%Y%m%d%H%M%S') + if end: + moment += datetime.timedelta(seconds=1) + except (ValueError, OverflowError): + raise UsageError(f'invalid timestamp {timestamp!r}') from None + return f"TIMESTAMP '{moment:%Y-%m-%d %H:%M:%S}'" + + +def check_language(code: str) -> str: + """An ISO 639-3 language code (three letters, e.g. eng or deu), in lowercase.""" + language = code.strip().lower() + if not re.fullmatch(r'[a-z]{3}', language): + raise UsageError(f'invalid language {code!r}, use ISO 639-3 codes (three letters), e.g. eng or deu') + return language + + +def url_surtkeys(query: IndexQuery) -> Tuple[List[str], List[str]]: + """--url patterns -> (url_surtkey prefixes, exact url_surtkey values).""" + prefixes, exact = [], [] + for pattern in query.urls: + p, e = url_pattern_to_surts(pattern) + prefixes += p + exact += e + return prefixes, exact + + +def query_conditions(query: IndexQuery) -> List[str]: + """WHERE predicates for --from, --to and --language.""" + conditions = [] + if query.from_ts: + conditions.append(f'fetch_time >= {sql_timestamp(query.from_ts)}') + if query.to: + conditions.append(f'fetch_time < {sql_timestamp(query.to, end=True)}') + if query.languages: + # primary content language: the first of the comma-separated content_languages + predicates = [] + for code in query.languages: + language = check_language(code) + predicates += [f"content_languages = '{language}'", f"content_languages LIKE '{language},%'"] + conditions.append('(' + ' OR '.join(predicates) + ')') + return conditions diff --git a/cdx_toolkit/indexes/resolve.py b/cdx_toolkit/indexes/resolve.py index e4c6a45..1eb17dd 100644 --- a/cdx_toolkit/indexes/resolve.py +++ b/cdx_toolkit/indexes/resolve.py @@ -2,7 +2,7 @@ from urllib.parse import urlparse from .base import IndexBackend, UsageError -from .cdx_api import CdxApiBackend +from .backends.cdx_api import CdxApiBackend # index names that map to CDX API sources known by CDXFetcher @@ -62,20 +62,20 @@ def resolve_index( location = INDEX_ALIASES.get(location, location) if index_format == 'athena': - from .sql import AthenaBackend + from .backends.athena import AthenaBackend return AthenaBackend(location, assume_yes=assume_yes) if index_format == 'duckdb': if location in INDEX_NAMES or is_cdx_api_location(location) or location.startswith('athena://'): raise UsageError(f'{location} cannot be queried with DuckDB; use a path to the parquet index') - from .sql import DuckDbBackend + from .backends.duckdb import DuckDbBackend return DuckDbBackend(location, assume_yes=assume_yes) if is_cdx_api_location(location): return CdxApiBackend(location, cc_mirror=cc_mirror, cc_sort=cc_sort) - from .cdx_files import CdxFilesBackend + from .backends.cdx_files import CdxFilesBackend return CdxFilesBackend(location) diff --git a/tests/unit/test_athena.py b/tests/unit/test_athena.py index 3aaf8c4..e525c2b 100644 --- a/tests/unit/test_athena.py +++ b/tests/unit/test_athena.py @@ -7,8 +7,9 @@ from cdx_toolkit.cli import main from cdx_toolkit.indexes import IndexQuery -from cdx_toolkit.indexes.sql import AthenaBackend -from cdx_toolkit.indexes.athena import iter_athena_rows, make_client, report_query_cost, run_athena_query +from cdx_toolkit.indexes.backends.athena import ( + AthenaBackend, iter_athena_rows, make_client, report_query_cost, run_athena_query, +) ATHENA = 'athena://athena.eu-west-1.amazonaws.com/mydb?S3OutputLocation=s3://bucket/athena/' COLUMNS = ['url', 'fetch_time', 'fetch_status', 'warc_filename', 'warc_record_offset', 'warc_record_length'] @@ -57,8 +58,8 @@ def get_paginator(self, name): @pytest.fixture def client(): fake = FakeAthenaClient() - with patch('cdx_toolkit.indexes.athena.make_client', return_value=fake), \ - patch('cdx_toolkit.indexes.athena.time.sleep'): + with patch('cdx_toolkit.indexes.backends.athena.make_client', return_value=fake), \ + patch('cdx_toolkit.indexes.backends.athena.time.sleep'): yield fake @@ -89,8 +90,8 @@ def test_athena_count(): fake.get_paginator = lambda name: type('P', (), {'paginate': lambda self, **kw: iter([ {'ResultSet': {'Rows': [{'Data': [{'VarCharValue': 'n'}]}, {'Data': [{'VarCharValue': '42'}]}]}} ])})() - with patch('cdx_toolkit.indexes.athena.make_client', return_value=fake), \ - patch('cdx_toolkit.indexes.athena.time.sleep'): + with patch('cdx_toolkit.indexes.backends.athena.make_client', return_value=fake), \ + patch('cdx_toolkit.indexes.backends.athena.time.sleep'): assert AthenaBackend(ATHENA, assume_yes=True).count(IndexQuery(domains=['example.com'])) == 42 assert fake.started[0]['QueryString'].startswith('SELECT count(*) AS n FROM (') @@ -105,7 +106,7 @@ def test_failed_query(caplog): def test_query_timeout_is_cancelled(): fake = FakeAthenaClient(states=['RUNNING']) - with patch('cdx_toolkit.indexes.athena.time.sleep'), pytest.raises(TimeoutError): + with patch('cdx_toolkit.indexes.backends.athena.time.sleep'), pytest.raises(TimeoutError): run_athena_query(fake, 'SELECT 1', 'ccindex', 's3://b/p/', max_wait_time=0) assert fake.stopped == ['qid'] diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py index bb3c673..f08d5c8 100644 --- a/tests/unit/test_cdx_files.py +++ b/tests/unit/test_cdx_files.py @@ -6,7 +6,7 @@ from cdx_toolkit.cli import main from cdx_toolkit.indexes import IndexQuery, UsageError, iter_index_records -from cdx_toolkit.indexes.cdx_files import ( +from cdx_toolkit.indexes.backends.cdx_files import ( CdxFilesBackend, domain_entry_to_surts, load_url_patterns, url_pattern_to_surts, ) from cdx_toolkit.record_io import write_records diff --git a/tests/unit/test_confirm_cost.py b/tests/unit/test_confirm_cost.py index 4f4d19e..8eb46bf 100644 --- a/tests/unit/test_confirm_cost.py +++ b/tests/unit/test_confirm_cost.py @@ -2,7 +2,7 @@ import pytest -from cdx_toolkit.indexes.sql import confirm_cost +from cdx_toolkit.indexes.backends.sql import confirm_cost def est(n_crawls): @@ -22,34 +22,34 @@ def test_confirmed_flag_bypasses_prompt(): def test_large_crawl_set_non_tty_aborts(): - with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin: + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin: stdin.isatty.return_value = False with pytest.raises(SystemExit): confirm_cost(*est(11), confirmed=False) def test_unknown_crawls_non_tty_aborts(): - with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin: + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin: stdin.isatty.return_value = False with pytest.raises(SystemExit): confirm_cost(*est(None), confirmed=False) def test_tty_yes_proceeds(): - with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch('builtins.input', return_value='y'): + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin, patch('builtins.input', return_value='y'): stdin.isatty.return_value = True confirm_cost(*est(None), confirmed=False) # should not raise def test_tty_empty_answer_aborts(): - with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch('builtins.input', return_value=''): + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin, patch('builtins.input', return_value=''): stdin.isatty.return_value = True with pytest.raises(SystemExit): confirm_cost(*est(None), confirmed=False) def test_tty_no_answer_aborts(): - with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch('builtins.input', return_value='n'): + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin, patch('builtins.input', return_value='n'): stdin.isatty.return_value = True with pytest.raises(SystemExit): confirm_cost(*est(11), confirmed=False) diff --git a/tests/unit/test_duckdb.py b/tests/unit/test_duckdb.py index 8256d87..529708f 100644 --- a/tests/unit/test_duckdb.py +++ b/tests/unit/test_duckdb.py @@ -3,8 +3,8 @@ import pytest -from cdx_toolkit.indexes import duckdb as duckdb_source -from cdx_toolkit.indexes.duckdb import build_from_clause, iter_duckdb_rows +from cdx_toolkit.indexes.backends import duckdb as duckdb_source +from cdx_toolkit.indexes.backends.duckdb import build_from_clause, iter_duckdb_rows from tests.conftest import requires_duckdb diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py index e1ab59a..548693b 100644 --- a/tests/unit/test_indexes.py +++ b/tests/unit/test_indexes.py @@ -3,8 +3,8 @@ import pytest from cdx_toolkit.indexes import IndexQuery, UsageError, iter_index_records, resolve_index -from cdx_toolkit.indexes.cdx_api import CdxApiBackend -from cdx_toolkit.indexes.cdx_files import CdxFilesBackend +from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend +from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend from cdx_toolkit.indexes.resolve import is_cdx_api_location diff --git a/tests/unit/test_sql_query.py b/tests/unit/test_sql_builder.py similarity index 92% rename from tests/unit/test_sql_query.py rename to tests/unit/test_sql_builder.py index 711b7d5..168a47b 100644 --- a/tests/unit/test_sql_query.py +++ b/tests/unit/test_sql_builder.py @@ -1,7 +1,7 @@ """Building the guided SQL queries for the columnar index (no engine).""" import pytest -from cdx_toolkit.indexes.sql_query import build_sql, escape_sql_literal +from cdx_toolkit.indexes.backends.sql_builder import build_sql, escape_sql_literal def test_build_query_hostnames(): @@ -84,9 +84,9 @@ def test_build_query_rejects_injection_hostname(): def test_resolve_crawl_names(): from unittest.mock import patch - from cdx_toolkit.indexes.sql_query import resolve_crawl_names + from cdx_toolkit.indexes.backends.sql_builder import resolve_crawl_names endpoints = [f'https://index.commoncrawl.org/CC-MAIN-2024-{week}-index' for week in (30, 33, 38)] - with patch('cdx_toolkit.indexes.sql_query.get_cc_endpoints', return_value=endpoints): + with patch('cdx_toolkit.indexes.backends.sql_builder.get_cc_endpoints', return_value=endpoints): assert resolve_crawl_names('2') == ['CC-MAIN-2024-33', 'CC-MAIN-2024-38'] assert resolve_crawl_names('CC-MAIN-2024-30') == ['CC-MAIN-2024-30'] diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index c7355cd..64e0d59 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -6,11 +6,12 @@ from cdx_toolkit.cli import main from cdx_toolkit.indexes import IndexQuery, UsageError, resolve_index -from cdx_toolkit.indexes.cdx_api import CdxApiBackend -from cdx_toolkit.indexes.cdx_files import CdxFilesBackend -from cdx_toolkit.indexes.sql import ( - AthenaBackend, check_language, DuckDbBackend, normalize_row, parse_athena_location, query_conditions, -) +from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend +from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend +from cdx_toolkit.indexes.backends.athena import AthenaBackend, parse_athena_location +from cdx_toolkit.indexes.backends.duckdb import DuckDbBackend +from cdx_toolkit.indexes.backends.sql import normalize_row +from cdx_toolkit.indexes.backends.sql_builder import check_language, query_conditions from tests.conftest import requires_duckdb @@ -103,7 +104,7 @@ def test_duckdb_guided_query_prunes_crawls(): backend = DuckDbBackend('s3://bucket/index/') query = IndexQuery(hostnames=['www.example.com'], domains=['example.org'], crawl='CC-MAIN-2024-33', limit=5) - with patch('cdx_toolkit.indexes.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): + with patch('cdx_toolkit.indexes.backends.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): sql, n_crawls = backend.prepare(query) assert n_crawls == 1 @@ -146,7 +147,7 @@ def test_athena_guided_query_and_cost_guard(): assert 'FROM ccindex' in sql # not restricted to crawls -> confirmation needed, refused in non-interactive mode - with patch('cdx_toolkit.indexes.sql.sys.stdin') as stdin, patch.object(backend, 'iter_sql_rows') as rows: + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin, patch.object(backend, 'iter_sql_rows') as rows: stdin.isatty.return_value = False with pytest.raises(SystemExit, match='--yes'): list(backend.iter_records(query)) @@ -207,7 +208,7 @@ def parquet_index(tmp_path): @requires_duckdb def test_index_duckdb_end_to_end(parquet_index, tmp_path, capsys): output = tmp_path / 'out.jsonl' - with patch('cdx_toolkit.indexes.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): + with patch('cdx_toolkit.indexes.backends.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): main([ 'index', str(parquet_index), '--index-format', 'duckdb', '--domain', 'example.com', '--crawl', 'CC-MAIN-2024-33', '-o', str(output), diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py index 28622c5..d8edebb 100644 --- a/tests/unit/test_usage_errors.py +++ b/tests/unit/test_usage_errors.py @@ -9,13 +9,13 @@ from cdx_toolkit.cli import main from cdx_toolkit.indexes import IndexQuery, UsageError -from cdx_toolkit.indexes.cdx_api import CdxApiBackend -from cdx_toolkit.indexes.cdx_files import CdxFilesBackend, load_url_patterns +from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend +from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend, load_url_patterns from cdx_toolkit.indexes.patterns import domain_entry_to_surts, url_pattern_to_surts -from cdx_toolkit.indexes.sql import DuckDbBackend, sql_timestamp +from cdx_toolkit.indexes.backends.duckdb import DuckDbBackend +from cdx_toolkit.indexes.backends.sql_builder import quote_sql_string, sql_timestamp from cdx_toolkit.record_io import RecordWriter, import_pyarrow, read_records, write_records from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.indexes.sql_query import quote_sql_string from tests.conftest import TEST_DATA_PATH From 8380740306cada316721a6120ec46e831dcb2192 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 12:48:29 +0200 Subject: [PATCH 41/91] refactor(index): every backend turns the query into a plan (plan(), select(), count(), explain()) IndexQuery -> the backend's plan (ApiPlan, CdxFilesPlan, SqlPlan), which select() and count() run and explain() describes. IndexBackend.records() applies --limit and then --url-fgrep/--url-fgrepv for every backend (replaces iter_index_records and the limit countdowns of the CDX API and CDX files backends). SqlBackend's engine methods are abstract, and build_sql takes the IndexQuery instead of ten keyword arguments. CDX files are CdxFile tuples. --- cdx_toolkit/cli/index.py | 7 +- cdx_toolkit/indexes/__init__.py | 2 +- cdx_toolkit/indexes/backends/cdx_api.py | 108 ++++++++++++-------- cdx_toolkit/indexes/backends/cdx_files.py | 78 ++++++++------ cdx_toolkit/indexes/backends/sql.py | 58 ++++++----- cdx_toolkit/indexes/backends/sql_builder.py | 67 +++++------- cdx_toolkit/indexes/base.py | 35 +++++-- tests/unit/test_cdx_files.py | 15 ++- tests/unit/test_cli.py | 4 +- tests/unit/test_indexes.py | 18 ++-- tests/unit/test_sql_builder.py | 28 +++-- tests/unit/test_sql_indexes.py | 20 ++-- 12 files changed, 236 insertions(+), 204 deletions(-) diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index 32cc225..f295e68 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -1,7 +1,7 @@ from argparse import Namespace, RawDescriptionHelpFormatter import logging -from cdx_toolkit.indexes import IndexQuery, UsageError, iter_index_records, resolve_index +from cdx_toolkit.indexes import IndexQuery, UsageError, resolve_index from cdx_toolkit.indexes.resolve import INDEX_FORMATS from cdx_toolkit.record_io import ( FORMAT_CHOICES, canonical_format, format_from_path, import_pyarrow, open_text, write_records, @@ -250,7 +250,6 @@ def run(args: Namespace, cmdline: str): fh.write(f'{count}\n') return - n = write_records( - iter_index_records(backend, query), args.output, output_format, args.fields, fixed_schema=backend.fixed_schema - ) + records = backend.records(query) + n = write_records(records, args.output, output_format, args.fields, fixed_schema=backend.fixed_schema) LOGGER.info('wrote %d captures to %s', n, args.output) diff --git a/cdx_toolkit/indexes/__init__.py b/cdx_toolkit/indexes/__init__.py index 57c61ed..f3bb730 100644 --- a/cdx_toolkit/indexes/__init__.py +++ b/cdx_toolkit/indexes/__init__.py @@ -1,3 +1,3 @@ -from .base import IndexBackend, UsageError, iter_index_records # noqa: F401 +from .base import IndexBackend, QueryPlan, UsageError # noqa: F401 from .query import IndexQuery # noqa: F401 from .resolve import resolve_index # noqa: F401 diff --git a/cdx_toolkit/indexes/backends/cdx_api.py b/cdx_toolkit/indexes/backends/cdx_api.py index a6f453c..df9c3a2 100644 --- a/cdx_toolkit/indexes/backends/cdx_api.py +++ b/cdx_toolkit/indexes/backends/cdx_api.py @@ -1,10 +1,11 @@ +from dataclasses import dataclass import logging -from typing import Iterator, Mapping, Optional +from typing import Iterator, List, Mapping, Optional import cdx_toolkit from cdx_toolkit.commoncrawl import normalize_crawl -from ..base import IndexBackend, UsageError +from ..base import IndexBackend, QueryPlan, UsageError from ..query import IndexQuery @@ -32,18 +33,19 @@ def validate(self, query: IndexQuery) -> None: if query.crawl and self.source != 'cc': raise UsageError('--crawl can only be used with the Common Crawl index (cc)') - def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: - """The captures of all selectors; a capture selected by more than one is only returned once. + def plan(self, query: IndexQuery) -> 'ApiPlan': + requests = [ApiRequest(url, api_params(query, match_type)) for url, match_type in query.url_patterns()] + return ApiPlan(self.source, requests, query.limit, query.crawl) - With several selectors, --limit is applied here (after removing the duplicates), not by the server.""" + def select(self, query: IndexQuery) -> Iterator[Mapping]: + """The captures of all selectors; a capture selected by more than one is only returned once.""" + plan = self.plan(query) cdx = self._fetcher(query) - patterns = query.url_patterns() - remaining = query.limit - seen = set() if len(patterns) > 1 else None - for url, match_type in patterns: - params = self._params(query, match_type, limit=remaining if seen is None else None) + seen = set() if len(plan.requests) > 1 else None + for request in plan.requests: + params = request.params_with(plan.server_limit) # closest only makes sense for a single (non-paged) query - captures = cdx.get(url, **params) if query.closest else cdx.iter(url, **params) + captures = cdx.get(request.url, **params) if query.closest else cdx.iter(request.url, **params) for capture in captures: if seen is not None: identity = capture_identity(capture) @@ -51,30 +53,17 @@ def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: continue seen.add(identity) yield capture - if remaining is not None: - remaining -= 1 - if remaining <= 0: - return def count(self, query: IndexQuery) -> int: """Estimate from the number of index pages, at most --limit; overlapping selectors are counted more than once.""" cdx = self._fetcher(query) count = sum( - cdx.get_size_estimate(url, **self._params(query, match_type, limit=query.limit)) - for url, match_type in query.url_patterns() + cdx.get_size_estimate(request.url, **request.params_with(query.limit)) + for request in self.plan(query).requests ) return min(count, query.limit) if query.limit else count - def explain(self, query: IndexQuery) -> str: - lines = [] - for url, match_type in query.url_patterns(): - params = ' '.join(f'{k}={v}' for k, v in self._params(query, match_type, limit=query.limit).items()) - lines.append(f'{self.source}: url={url} {params}'.rstrip()) - if query.crawl: - lines.append(f'crawls: {query.crawl}') - return '\n'.join(lines) + '\n' - def _fetcher(self, query: IndexQuery) -> cdx_toolkit.CDXFetcher: kwargs = {'source': self.source} if self.cc_mirror: @@ -85,22 +74,57 @@ def _fetcher(self, query: IndexQuery) -> cdx_toolkit.CDXFetcher: kwargs['crawl'] = normalize_crawl([query.crawl]) return cdx_toolkit.CDXFetcher(**kwargs) - @staticmethod - def _params(query: IndexQuery, match_type: Optional[str], limit: Optional[int]) -> dict: - params = {} - if limit: - params['limit'] = limit - if query.from_ts: - params['from_ts'] = query.from_ts - if query.to: - params['to'] = query.to - if query.closest: - params['closest'] = query.closest - if query.filters: - params['filter'] = list(query.filters) - if match_type: - params['matchType'] = match_type - return params + +@dataclass +class ApiRequest: + """One request (series of paged requests) to the CDX API: a URL and its parameters (without the limit).""" + + url: str + params: dict + + def params_with(self, limit: Optional[int]) -> dict: + return {'limit': limit, **self.params} if limit else dict(self.params) + + +@dataclass +class ApiPlan(QueryPlan): + """One request per URL selector (--url, --hostname, --domain).""" + + source: str + requests: List[ApiRequest] + limit: Optional[int] = None + crawl: Optional[str] = None + + @property + def server_limit(self) -> Optional[int]: + """The limit sent to the server: only for a single request; with several, the captures selected by more + than one request are removed first (and the limit is applied by IndexBackend.records()).""" + return self.limit if len(self.requests) == 1 else None + + def describe(self) -> str: + lines = [] + for request in self.requests: + params = ' '.join(f'{k}={v}' for k, v in request.params_with(self.limit).items()) + lines.append(f'{self.source}: url={request.url} {params}'.rstrip()) + if self.crawl: + lines.append(f'crawls: {self.crawl}') + return '\n'.join(lines) + '\n' + + +def api_params(query: IndexQuery, match_type: Optional[str]) -> dict: + """The parameters of a CDX API request (CDXFetcher.iter/get) for one URL selector.""" + params = {} + if query.from_ts: + params['from_ts'] = query.from_ts + if query.to: + params['to'] = query.to + if query.closest: + params['closest'] = query.closest + if query.filters: + params['filter'] = list(query.filters) + if match_type: + params['matchType'] = match_type + return params def capture_identity(capture: Mapping) -> tuple: diff --git a/cdx_toolkit/indexes/backends/cdx_files.py b/cdx_toolkit/indexes/backends/cdx_files.py index d354be1..8697d6c 100644 --- a/cdx_toolkit/indexes/backends/cdx_files.py +++ b/cdx_toolkit/indexes/backends/cdx_files.py @@ -9,15 +9,15 @@ import os from concurrent.futures import ProcessPoolExecutor from dataclasses import dataclass -from typing import Iterator, List, Mapping, Optional, Tuple +from itertools import islice +from typing import Iterator, List, Mapping, NamedTuple, Optional, Tuple from cdx_toolkit.record_io import check_glob, format_from_path, is_glob, open_text, parse_cdxj_line, read_records from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up -from ..base import IndexBackend, UsageError -from ..patterns import ( # noqa: F401 (url_pattern_to_surts, to_surt) - domain_entry_to_surts, domain_to_surts, hostname_to_surts, normalize_host, to_surt, url_hostname, - url_pattern_to_surts, +from ..base import IndexBackend, QueryPlan, UsageError +from ..patterns import ( + domain_entry_to_surts, domain_to_surts, hostname_to_surts, normalize_host, url_hostname, url_pattern_to_surts, ) from ..query import IndexQuery @@ -210,48 +210,35 @@ def validate(self, query: IndexQuery) -> None: for pattern in query.urls: url_pattern_to_surts(pattern) - def files(self) -> List[Tuple[str, str]]: - """(path, path relative to the base directory of the glob) of each CDX file.""" + def files(self) -> List['CdxFile']: + """The CDX files of the location (a file or a glob).""" import fsspec if not is_glob(self.location): if '://' in self.location: - return [(self.location, self.location.split('?', 1)[0].rstrip('/').rsplit('/', 1)[-1])] - return [(self.location, os.path.basename(self.location))] # also with backslashes on Windows + return [CdxFile(self.location, self.location.split('?', 1)[0].rstrip('/').rsplit('/', 1)[-1])] + return [CdxFile(self.location, os.path.basename(self.location))] # also with backslashes on Windows fs, fs_path = fsspec.url_to_fs(self.location) first_glob = min(fs_path.index(c) for c in GLOB_CHARS if c in fs_path) base = fs_path[:fs_path.rfind('/', 0, first_glob)] paths = sorted(fs.glob(fs_path)) if not paths: raise FileNotFoundError(f'no files found for {self.location}') - return [(fs.unstrip_protocol(p), p[len(base):].lstrip('/')) for p in paths] + return [CdxFile(fs.unstrip_protocol(p), p[len(base):].lstrip('/')) for p in paths] - def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: - matcher = CaptureFilter.from_query(query).matcher() - remaining = query.limit - for path, _ in self.files(): - for line in iter_matching_lines(path, matcher): + def plan(self, query: IndexQuery) -> 'CdxFilesPlan': + return CdxFilesPlan(self.location, self.files(), CaptureFilter.from_query(query)) + + def select(self, query: IndexQuery) -> Iterator[Mapping]: + plan = self.plan(query) + matcher = plan.capture_filter.matcher() + for cdx_file in plan.files: + for line in iter_matching_lines(cdx_file.path, matcher): yield parse_cdxj_line(line) - if remaining is not None: - remaining -= 1 - if remaining <= 0: - return def count(self, query: IndexQuery) -> int: """Exact number of selected captures, at most --limit (reads all files, or until the limit).""" - return sum(1 for _ in self.iter_records(query)) - - def explain(self, query: IndexQuery) -> str: - capture_filter = CaptureFilter.from_query(query) - files = self.files() - lines = [f'{len(files)} CDX file(s): {self.location}'] - lines += [f' {path}' for path, _ in files[:20]] + ([' ...'] if len(files) > 20 else []) - lines.append(f'{len(capture_filter.prefixes)} SURT prefix(es), {len(capture_filter.exact)} exact URL(s)') - lines += [f' prefix {p}' for p in capture_filter.prefixes[:20]] - lines += [f' exact {e}' for e in capture_filter.exact[:20]] - if capture_filter.from_ts or capture_filter.to: - lines.append(f'timestamps: {capture_filter.from_ts or "-"} to {capture_filter.to or "-"}') - return '\n'.join(lines) + '\n' + return sum(1 for _ in islice(self.select(query), query.limit)) def write_per_file(self, query: IndexQuery, output_base: str, processes: int = 1) -> Tuple[int, int]: """Write the selected lines of each CDX file, unchanged, to /. @@ -262,7 +249,7 @@ def write_per_file(self, query: IndexQuery, output_base: str, processes: int = 1 capture_filter = CaptureFilter.from_query(query, url_filters=True) join = (lambda rel: output_base.rstrip('/') + '/' + rel) if '://' in output_base else \ (lambda rel: os.path.join(output_base, *rel.split('/'))) - tasks = [(path, join(rel), capture_filter) for path, rel in self.files()] + tasks = [(f.path, join(f.relative_path), capture_filter) for f in self.files()] if processes > 1: with ProcessPoolExecutor(max_workers=processes) as ex: @@ -273,3 +260,28 @@ def write_per_file(self, query: IndexQuery, output_base: str, processes: int = 1 written = [(path, n) for path, n in results if n] LOGGER.info('wrote %d captures to %d of %d files', sum(n for _, n in written), len(written), len(tasks)) return len(written), sum(n for _, n in written) + + +class CdxFile(NamedTuple): + path: str + relative_path: str # to the base directory of the glob (the path of the output with one output per file) + + +@dataclass +class CdxFilesPlan(QueryPlan): + """The CDX files and which of their lines are selected.""" + + location: str + files: List[CdxFile] + capture_filter: CaptureFilter + + def describe(self) -> str: + f = self.capture_filter + lines = [f'{len(self.files)} CDX file(s): {self.location}'] + lines += [f' {cdx_file.path}' for cdx_file in self.files[:20]] + ([' ...'] if len(self.files) > 20 else []) + lines.append(f'{len(f.prefixes)} SURT prefix(es), {len(f.exact)} exact URL(s)') + lines += [f' prefix {p}' for p in f.prefixes[:20]] + lines += [f' exact {e}' for e in f.exact[:20]] + if f.from_ts or f.to: + lines.append(f'timestamps: {f.from_ts or "-"} to {f.to or "-"}') + return '\n'.join(lines) + '\n' diff --git a/cdx_toolkit/indexes/backends/sql.py b/cdx_toolkit/indexes/backends/sql.py index e5b57b8..f62edc3 100644 --- a/cdx_toolkit/indexes/backends/sql.py +++ b/cdx_toolkit/indexes/backends/sql.py @@ -1,13 +1,15 @@ ''' The common logic of the backends for the columnar URL index (parquet), queried with SQL (see duckdb.py, athena.py). ''' +from abc import abstractmethod +from dataclasses import dataclass import datetime import logging import re import sys from typing import Iterator, List, Mapping, Optional, Tuple -from ..base import IndexBackend, UsageError +from ..base import IndexBackend, QueryPlan, UsageError from ..query import IndexQuery, field_flag from .sql_builder import build_sql, query_conditions, resolve_crawl_names, url_surtkeys @@ -127,47 +129,49 @@ def validate(self, query: IndexQuery) -> None: url_surtkeys(query) # check --url query_conditions(query) # check --from, --to and --language + @abstractmethod def from_clause(self, crawls: Optional[List[str]]) -> Tuple[str, bool]: """FROM expression for the guided query, and whether it already prunes the crawls.""" raise NotImplementedError # pragma: no cover + @abstractmethod def iter_sql_rows(self, sql: str) -> Iterator[dict]: + """Run the SQL and yield the result rows as dicts (column name -> value).""" raise NotImplementedError # pragma: no cover - def prepare(self, query: IndexQuery, order_by: bool = True) -> Tuple[str, Optional[int]]: - """The SQL to run and the number of crawls it is restricted to (None = unknown).""" + def plan(self, query: IndexQuery, order_by: bool = True) -> 'SqlPlan': + """The SQL to run (--sql, or the guided query) and the number of crawls it is restricted to.""" if query.sql: - return query.sql.strip().rstrip(';'), None + return SqlPlan(self.name, self.location, query.sql.strip().rstrip(';'), n_crawls=None) crawls = resolve_crawl_names(query.crawl) if query.crawl else None from_clause, prunes_crawls = self.from_clause(crawls) - surtkey_prefixes, surtkeys = url_surtkeys(query) sql = build_sql( - from_clause, - query.hostnames, - crawls=None if prunes_crawls else crawls, - limit=query.limit or 0, - url_host_registered_domains=query.domains, - order_by=order_by, - columns=list(COLUMNS), - conditions=query_conditions(query), - url_surtkey_prefixes=surtkey_prefixes, - url_surtkeys=surtkeys, + query, from_clause, crawls=None if prunes_crawls else crawls, order_by=order_by, columns=list(COLUMNS), ) - return sql, len(crawls) if crawls else None + return SqlPlan(self.name, self.location, sql, n_crawls=len(crawls) if crawls else None) - def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: - sql, n_crawls = self.prepare(query) - confirm_cost(n_crawls, self.engine, self.assume_yes) - for row in self.iter_sql_rows(sql): + def select(self, query: IndexQuery) -> Iterator[Mapping]: + plan = self.plan(query) + confirm_cost(plan.n_crawls, self.engine, self.assume_yes) + for row in self.iter_sql_rows(plan.sql): yield normalize_row(row) def count(self, query: IndexQuery) -> int: - sql, n_crawls = self.prepare(query, order_by=False) - confirm_cost(n_crawls, self.engine, self.assume_yes) - row = next(self.iter_sql_rows(f'SELECT count(*) AS n FROM ({sql}) AS q')) + plan = self.plan(query, order_by=False) + confirm_cost(plan.n_crawls, self.engine, self.assume_yes) + row = next(self.iter_sql_rows(f'SELECT count(*) AS n FROM ({plan.sql}) AS q')) return int(row['n']) - def explain(self, query: IndexQuery) -> str: - sql, n_crawls = self.prepare(query) - scope = f'{n_crawls} crawl(s)' if n_crawls else 'all crawls (or not verifiable)' - return f'-- {self.name} query on {self.location}, restricted to {scope}\n{sql.strip()}\n' + +@dataclass +class SqlPlan(QueryPlan): + """The SQL of a query and the number of crawls it is restricted to (None: all crawls, or unknown for --sql).""" + + backend_name: str + location: str + sql: str + n_crawls: Optional[int] + + def describe(self) -> str: + scope = f'{self.n_crawls} crawl(s)' if self.n_crawls else 'all crawls (or not verifiable)' + return f'-- {self.backend_name} query on {self.location}, restricted to {scope}\n{self.sql.strip()}\n' diff --git a/cdx_toolkit/indexes/backends/sql_builder.py b/cdx_toolkit/indexes/backends/sql_builder.py index 80bac8a..fca0926 100644 --- a/cdx_toolkit/indexes/backends/sql_builder.py +++ b/cdx_toolkit/indexes/backends/sql_builder.py @@ -71,37 +71,26 @@ def domain_predicate(domain: str) -> str: return f'(url_host_registered_domain IN ({registered}) AND ({surt_match}))' -def build_where_sql( - url_host_names: Optional[List[str]] = None, - crawls: Optional[List[str]] = None, - url_host_registered_domains: Optional[List[str]] = None, - conditions: Optional[List[str]] = None, - url_surtkey_prefixes: Optional[List[str]] = None, - url_surtkeys: Optional[List[str]] = None, -) -> str: - """Build the WHERE body (without the `WHERE` keyword) shared by all SQL engines. - - The guided filter matches on `url_host_name` (exact host, e.g. www.example.com), - domains with their subdomains (see domain_predicate) and/or `url_surtkey` (SURT keys - of URL patterns: `url_surtkey_prefixes` match all keys starting with them, - `url_surtkeys` exactly); predicates are OR-ed together. - At least one host, domain or SURT key is required. If `crawls` is a non-empty list - of crawl names (e.g. ['CC-MAIN-2025-33']), a `crawl IN (...)` partition filter is - added -- the main lever for reducing scan cost. `conditions` are further (already - escaped) predicates that are AND-ed. Engines differ only in their FROM clause (see +def build_where_sql(query: IndexQuery, crawls: Optional[List[str]] = None) -> str: + """Build the WHERE body (without the `WHERE` keyword) of a guided query, shared by all SQL engines. + + The URL selectors are OR-ed: --hostname matches `url_host_name` (exact host, e.g. www.example.com), + --domain the domain with its subdomains (see domain_predicate), --url the `url_surtkey` (SURT prefixes or + exact keys). At least one of them is required. If `crawls` is a non-empty list of crawl names (e.g. + ['CC-MAIN-2025-33']), a `crawl IN (...)` partition filter is added -- the main lever for reducing scan cost. + --from, --to and --language are AND-ed (see query_conditions). Engines differ only in their FROM clause (see build_sql).""" - url_host_names = [normalize_host(h) for h in url_host_names or []] - domains = [normalize_host(d) for d in url_host_registered_domains or []] - surt_prefixes = url_surtkey_prefixes or [] - surts = url_surtkeys or [] - if not url_host_names and not domains and not surt_prefixes and not surts: + hostnames = [normalize_host(h) for h in query.hostnames] + domains = [normalize_host(d) for d in query.domains] + surt_prefixes, surts = url_surtkeys(query) + if not hostnames and not domains and not surt_prefixes and not surts: raise ValueError('an index query requires at least one hostname, registered domain or URL') - tlds = {v.split('.')[-1] for v in (list(url_host_names) + list(domains))} + tlds = {v.split('.')[-1] for v in hostnames + domains} tlds |= {surt_tld(s) for s in surt_prefixes + surts} query_tlds = ' OR '.join(f'url_host_tld = {escape_sql_literal(t)}' for t in sorted(tlds)) - host_predicates = [f'url_host_name = {escape_sql_literal(h)}' for h in url_host_names] + host_predicates = [f'url_host_name = {escape_sql_literal(h)}' for h in hostnames] host_predicates += [domain_predicate(d) for d in domains] host_predicates += [f'url_surtkey = {quote_sql_string(s)}' for s in surts] host_predicates += [f'url_surtkey {sql_like_prefix(s)}' for s in surt_prefixes] @@ -115,7 +104,7 @@ def build_where_sql( if crawls: crawl_in = ', '.join(escape_sql_literal(c) for c in crawls) clauses.append(f'crawl IN ({crawl_in})') - clauses.extend(conditions or []) + clauses.extend(query_conditions(query)) return '\n AND '.join(clauses) @@ -127,29 +116,21 @@ def build_where_sql( def build_sql( + query: IndexQuery, from_clause: str, - url_host_names: Optional[List[str]] = None, crawls: Optional[List[str]] = None, - limit: int = 0, - url_host_registered_domains: Optional[List[str]] = None, order_by: bool = True, columns: Optional[List[str]] = None, - conditions: Optional[List[str]] = None, - url_surtkey_prefixes: Optional[List[str]] = None, - url_surtkeys: Optional[List[str]] = None, ) -> str: - """Assemble a full SELECT for the columnar index. - - `from_clause` is the text following FROM (e.g. `ccindex` for Athena, or a - `read_parquet(...)` expression for DuckDB). When `order_by` is set, results are - sorted by (warc_filename, warc_record_offset) for fetch-time read locality. - `columns` defaults to the WARC location columns.""" - where_sql = build_where_sql( - url_host_names, crawls, url_host_registered_domains=url_host_registered_domains, conditions=conditions, - url_surtkey_prefixes=url_surtkey_prefixes, url_surtkeys=url_surtkeys, - ) + """The SELECT of a guided query (--url, --hostname, --domain, ...) of the columnar index. + + `from_clause` is the text following FROM (e.g. `ccindex` for Athena, or a `read_parquet(...)` expression for + DuckDB); `crawls` adds a partition filter (see build_where_sql). When `order_by` is set, results are sorted by + (warc_filename, warc_record_offset) for fetch-time read locality. `columns` defaults to the WARC location + columns.""" + where_sql = build_where_sql(query, crawls) order_sql = f'\n {ORDER_BY_SQL}' if order_by else '' - limit_sql = f'\n LIMIT {limit}' if limit and limit > 0 else '' + limit_sql = f'\n LIMIT {query.limit}' if query.limit and query.limit > 0 else '' columns_sql = ', '.join(columns or ('warc_filename', 'warc_record_offset', 'warc_record_length')) return f""" diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py index 479c6b3..dcb773c 100644 --- a/cdx_toolkit/indexes/base.py +++ b/cdx_toolkit/indexes/base.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from itertools import islice import logging from typing import FrozenSet, Iterable, Iterator, Mapping @@ -12,8 +13,19 @@ class UsageError(ValueError): """Invalid combination of command line options, reported without a traceback.""" +class QueryPlan(ABC): + """A query in the form of one backend, e.g. SQL text or SURT prefixes; what --dry-run shows.""" + + @abstractmethod + def describe(self) -> str: + raise NotImplementedError # pragma: no cover + + class IndexBackend(ABC): - """An index that can be queried for captures (CDX API, and later CDX files, Athena, DuckDB).""" + """An index that can be queried for captures: the CDX API, CDX files, or the columnar index (Athena, DuckDB). + + A query (IndexQuery, the same for every backend) becomes the backend's plan (plan()), which is run by + select() and count(), or shown by explain().""" name: str = '' supported_options: FrozenSet[str] = frozenset() @@ -29,10 +41,15 @@ def validate(self, query: IndexQuery) -> None: raise UsageError(f'--limit must be at least 1, not {query.limit}') @abstractmethod - def iter_records(self, query: IndexQuery) -> Iterator[Mapping]: + def plan(self, query: IndexQuery) -> QueryPlan: + """The query in the form of this backend.""" + raise NotImplementedError # pragma: no cover + + @abstractmethod + def select(self, query: IndexQuery) -> Iterator[Mapping]: """Yield the selected captures as mappings with CDX field names (url, timestamp, filename, ...). - Backends apply --limit themselves, so it can be pushed down to the index.""" + A backend can push --limit down to the index, but need not stop at it (see records()).""" raise NotImplementedError # pragma: no cover @abstractmethod @@ -41,15 +58,13 @@ def count(self, query: IndexQuery) -> int: client-side --url-fgrep/--url-fgrepv, which cannot be combined with --count).""" raise NotImplementedError # pragma: no cover - @abstractmethod + def records(self, query: IndexQuery) -> Iterator[Mapping]: + """The captures of the query: at most --limit, then filtered by --url-fgrep/--url-fgrepv (client-side).""" + return filter_by_url(islice(self.select(query), query.limit), query.url_fgrep, query.url_fgrepv) + def explain(self, query: IndexQuery) -> str: """Describe what would be queried (--dry-run), without querying the index.""" - raise NotImplementedError # pragma: no cover - - -def iter_index_records(backend: IndexBackend, query: IndexQuery) -> Iterator[Mapping]: - """Query a backend and apply the client-side filters that work for every backend.""" - return filter_by_url(backend.iter_records(query), query.url_fgrep, query.url_fgrepv) + return self.plan(query).describe() def filter_by_url(records: Iterable[Mapping], url_fgrep=None, url_fgrepv=None) -> Iterator[Mapping]: diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py index f08d5c8..4fb0085 100644 --- a/tests/unit/test_cdx_files.py +++ b/tests/unit/test_cdx_files.py @@ -5,10 +5,9 @@ import pytest from cdx_toolkit.cli import main -from cdx_toolkit.indexes import IndexQuery, UsageError, iter_index_records -from cdx_toolkit.indexes.backends.cdx_files import ( - CdxFilesBackend, domain_entry_to_surts, load_url_patterns, url_pattern_to_surts, -) +from cdx_toolkit.indexes import IndexQuery, UsageError +from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend, load_url_patterns +from cdx_toolkit.indexes.patterns import domain_entry_to_surts, url_pattern_to_surts from cdx_toolkit.record_io import write_records from tests.conftest import TEST_DATA_PATH @@ -52,7 +51,7 @@ def test_select_by_hostname_domain_and_url(): assert backend.count(IndexQuery(urls=['biodiv.mnhn.fr/fr/taxonomy*'])) == count_lines('fr,mnhn,biodiv)/fr/taxonomy') exact_key = LINES[0].split(' ')[0] - records = list(backend.iter_records(IndexQuery(urls=[json.loads(LINES[0].split(' ', 2)[2])['url']]))) + records = list(backend.records(IndexQuery(urls=[json.loads(LINES[0].split(' ', 2)[2])['url']]))) assert {r['urlkey'] for r in records} == {exact_key} @@ -61,7 +60,7 @@ def test_time_range_fgrep_and_limit(): query = IndexQuery(domains=['missiondefrance.fr'], from_ts='20240716', to='20240716') assert backend.count(query) == count_lines('fr,missiondefrance,', day='20240716') - records = list(iter_index_records(backend, IndexQuery(domains=['mnhn.fr'], url_fgrep='/taxonomy/', limit=5))) + records = list(backend.records(IndexQuery(domains=['mnhn.fr'], url_fgrep='/taxonomy/', limit=5))) assert 0 < len(records) <= 5 assert all('/taxonomy/' in r['url'] and r['urlkey'].startswith('fr,mnhn,biodiv)') for r in records) assert set(records[0]) >= {'urlkey', 'timestamp', 'url', 'filename', 'offset', 'length'} @@ -75,9 +74,9 @@ def test_limit_applies_before_the_url_filters(tmp_path, capsys): )) backend = CdxFilesBackend(str(path)) query = IndexQuery(domains=['example.com'], limit=1, url_fgrep='keep') - assert list(iter_index_records(backend, query)) == [] + assert list(backend.records(query)) == [] query.url_fgrep, query.url_fgrepv = None, 'drop' - assert list(iter_index_records(backend, query)) == [] + assert list(backend.records(query)) == [] assert backend.count(IndexQuery(domains=['example.com'], limit=1)) == 1 assert backend.count(IndexQuery(domains=['example.com'])) == 2 diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 612a326..9451723 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -136,8 +136,8 @@ def test_index_to_parquet_and_repackage(tmp_path, capsys): pytest.importorskip('pyarrow') captures = tmp_path / 'captures.parquet' - with patch.object(cdx_toolkit.cli.index, 'iter_index_records', return_value=iter([CC_CAPTURE])), \ - patch.object(cdx_toolkit.cli.index, 'resolve_index'): + with patch.object(cdx_toolkit.cli.index, 'resolve_index') as resolve_index: + resolve_index.return_value.records.return_value = iter([CC_CAPTURE]) main(['index', 'cc', '--url', 'example.com/*', '-o', str(captures)]) with pytest.raises(SystemExit): main(['index', 'cc', '--url', 'example.com/*', '--output-format', 'parquet']) diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py index 548693b..30d54bb 100644 --- a/tests/unit/test_indexes.py +++ b/tests/unit/test_indexes.py @@ -2,7 +2,7 @@ import pytest -from cdx_toolkit.indexes import IndexQuery, UsageError, iter_index_records, resolve_index +from cdx_toolkit.indexes import IndexQuery, UsageError, resolve_index from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend from cdx_toolkit.indexes.resolve import is_cdx_api_location @@ -80,7 +80,7 @@ def test_cdx_api_url_patterns_and_limit(): with patch('cdx_toolkit.CDXFetcher') as fetcher: fetcher.return_value.iter.side_effect = lambda url, **params: iter(results[url]) - records = list(CdxApiBackend('cc').iter_records(query)) + records = list(CdxApiBackend('cc').records(query)) assert [r['url'] for r in records] == ['http://example.com/a', 'http://example.com/b', 'http://www.example.org/'] @@ -96,7 +96,7 @@ def test_cdx_api_url_patterns_and_limit(): def test_cdx_api_single_selector_limit_is_sent_to_the_server(): with patch('cdx_toolkit.CDXFetcher') as fetcher: fetcher.return_value.iter.return_value = iter([_capture('http://example.com/a')]) - list(CdxApiBackend('cc').iter_records(IndexQuery(urls=['example.com/*'], limit=3))) + list(CdxApiBackend('cc').records(IndexQuery(urls=['example.com/*'], limit=3))) assert fetcher.return_value.iter.call_args.kwargs == {'limit': 3} @@ -114,12 +114,12 @@ def capture(url, timestamp, offset): query = IndexQuery(urls=['example.com/*'], domains=['example.com']) with patch('cdx_toolkit.CDXFetcher') as fetcher: fetcher.return_value.iter.side_effect = lambda url, **params: iter(results[url]) - records = list(CdxApiBackend('cc').iter_records(query)) + records = list(CdxApiBackend('cc').records(query)) assert [r['offset'] for r in records] == ['1', '2', '3', '4'] # the limit counts distinct captures query.limit = 3 - records = list(CdxApiBackend('cc').iter_records(query)) + records = list(CdxApiBackend('cc').records(query)) assert [r['offset'] for r in records] == ['1', '2', '3'] @@ -128,7 +128,7 @@ def test_cdx_api_closest_uses_get_and_crawl_is_normalized(): with patch('cdx_toolkit.CDXFetcher') as fetcher: fetcher.return_value.get.return_value = [_capture('http://example.com/')] - records = list(CdxApiBackend('cc').iter_records(query)) + records = list(CdxApiBackend('cc').records(query)) assert len(records) == 1 assert fetcher.call_args.kwargs == {'source': 'cc', 'crawl': ['CC-MAIN-2024-33', 'CC-MAIN-2024-30']} @@ -149,7 +149,7 @@ def test_cdx_api_count(): assert CdxApiBackend('ia').count(query) == 4500 -def test_iter_index_records_url_fgrep(): +def test_records_url_fgrep(): backend = CdxApiBackend('cc') query = IndexQuery(urls=['example.com/*'], url_fgrep='/keep', url_fgrepv='robots.txt') captures = [ @@ -158,7 +158,7 @@ def test_iter_index_records_url_fgrep(): _capture('http://example.com/keep/this'), ] - with patch.object(backend, 'iter_records', return_value=iter(captures)): - records = list(iter_index_records(backend, query)) + with patch.object(backend, 'select', return_value=iter(captures)): + records = list(backend.records(query)) assert [r['url'] for r in records] == ['http://example.com/keep/this'] diff --git a/tests/unit/test_sql_builder.py b/tests/unit/test_sql_builder.py index 168a47b..04de22a 100644 --- a/tests/unit/test_sql_builder.py +++ b/tests/unit/test_sql_builder.py @@ -1,11 +1,12 @@ """Building the guided SQL queries for the columnar index (no engine).""" import pytest +from cdx_toolkit.indexes import IndexQuery from cdx_toolkit.indexes.backends.sql_builder import build_sql, escape_sql_literal def test_build_query_hostnames(): - q = build_sql('ccindex', ['example.com', 'test.org']) + q = build_sql(IndexQuery(hostnames=['example.com', 'test.org']), 'ccindex') assert "subset = 'warc'" in q assert "url_host_tld = 'com'" in q assert "url_host_tld = 'org'" in q @@ -17,14 +18,14 @@ def test_build_query_hostnames(): def test_build_query_domains_only(): - q = build_sql('ccindex', url_host_registered_domains=['example.com']) + q = build_sql(IndexQuery(domains=['example.com']), 'ccindex') assert "url_host_registered_domain IN ('example.com')" in q assert "url_host_tld = 'com'" in q assert 'url_host_name' not in q def test_build_query_hostnames_and_domains(): - q = build_sql('ccindex', ['www.example.com'], url_host_registered_domains=['example.org']) + q = build_sql(IndexQuery(hostnames=['www.example.com'], domains=['example.org']), 'ccindex') assert "url_host_name = 'www.example.com'" in q assert "url_host_registered_domain IN ('example.org')" in q assert "url_host_tld = 'com'" in q @@ -33,38 +34,33 @@ def test_build_query_hostnames_and_domains(): def test_build_query_requires_host_or_domain(): with pytest.raises(ValueError): - build_sql('ccindex') + build_sql(IndexQuery(), 'ccindex') def test_build_query_with_crawls(): - q = build_sql('ccindex', ['example.com'], crawls=['CC-MAIN-2025-33', 'CC-MAIN-2025-30']) + q = build_sql(IndexQuery(hostnames=['example.com']), 'ccindex', crawls=['CC-MAIN-2025-33', 'CC-MAIN-2025-30']) assert "crawl IN ('CC-MAIN-2025-33', 'CC-MAIN-2025-30')" in q def test_build_query_limit(): - assert 'LIMIT 10' in build_sql('ccindex', ['example.com'], limit=10) - assert 'LIMIT' not in build_sql('ccindex', ['example.com'], limit=0) + assert 'LIMIT 10' in build_sql(IndexQuery(hostnames=['example.com'], limit=10), 'ccindex') + assert 'LIMIT' not in build_sql(IndexQuery(hostnames=['example.com']), 'ccindex') def test_build_query_orders_by_default(): - q = build_sql('ccindex', ['example.com']) + q = build_sql(IndexQuery(hostnames=['example.com']), 'ccindex') assert 'ORDER BY warc_filename, warc_record_offset' in q def test_build_query_order_by_false(): - assert 'ORDER BY' not in build_sql('ccindex', ['example.com'], order_by=False) + assert 'ORDER BY' not in build_sql(IndexQuery(hostnames=['example.com']), 'ccindex', order_by=False) def test_build_query_order_by_precedes_limit(): - q = build_sql('ccindex', ['example.com'], limit=10) + q = build_sql(IndexQuery(hostnames=['example.com'], limit=10), 'ccindex') assert q.index('ORDER BY') < q.index('LIMIT') -def test_build_query_requires_hostnames(): - with pytest.raises(ValueError): - build_sql('ccindex', []) - - def test_escape_sql_literal_valid(): assert escape_sql_literal('example.com') == "'example.com'" assert escape_sql_literal('CC-MAIN-2025-33') == "'CC-MAIN-2025-33'" @@ -78,7 +74,7 @@ def test_escape_sql_literal_rejects_injection(bad): def test_build_query_rejects_injection_hostname(): with pytest.raises(ValueError): - build_sql('ccindex', ["example.com'; DROP TABLE ccindex; --"]) + build_sql(IndexQuery(hostnames=["example.com'; DROP TABLE ccindex; --"]), 'ccindex') def test_resolve_crawl_names(): diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index 64e0d59..b963239 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -105,9 +105,10 @@ def test_duckdb_guided_query_prunes_crawls(): query = IndexQuery(hostnames=['www.example.com'], domains=['example.org'], crawl='CC-MAIN-2024-33', limit=5) with patch('cdx_toolkit.indexes.backends.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): - sql, n_crawls = backend.prepare(query) + plan = backend.plan(query) + sql = plan.sql - assert n_crawls == 1 + assert plan.n_crawls == 1 assert "read_parquet(['s3://bucket/index/crawl=CC-MAIN-2024-33/subset=warc/*.parquet']" in sql assert 'crawl IN' not in sql # pruned by the glob assert ("url_host_name = 'www.example.com' OR (url_host_registered_domain IN ('example.org') " @@ -119,7 +120,7 @@ def test_duckdb_guided_query_prunes_crawls(): def test_guided_query_url_patterns(): query = IndexQuery(urls=['example.com/a_b*', 'https://www.example.org/', '*.example.net'], domains=['example.com']) - sql, _ = AthenaBackend(ATHENA).prepare(query) + sql = AthenaBackend(ATHENA).plan(query).sql assert "(url_host_tld = 'com' OR url_host_tld = 'net' OR url_host_tld = 'org')" in sql assert ("((url_host_registered_domain IN ('example.com') AND (url_surtkey LIKE 'com,example)%' ESCAPE '\\' " @@ -130,7 +131,7 @@ def test_guided_query_url_patterns(): def test_guided_query_time_range_and_languages(): query = IndexQuery(hostnames=['example.com'], from_ts='2024', to='20240410', languages=['deu', 'fra']) - sql, _ = AthenaBackend(ATHENA).prepare(query) + sql = AthenaBackend(ATHENA).plan(query).sql assert "fetch_time >= TIMESTAMP '2024-01-01 00:00:00'" in sql assert "fetch_time < TIMESTAMP '2024-04-11 00:00:00'" in sql @@ -142,20 +143,21 @@ def test_athena_guided_query_and_cost_guard(): backend = AthenaBackend(ATHENA) query = IndexQuery(domains=['example.com']) - sql, n_crawls = backend.prepare(query) - assert n_crawls is None + plan = backend.plan(query) + sql = plan.sql + assert plan.n_crawls is None assert 'FROM ccindex' in sql # not restricted to crawls -> confirmation needed, refused in non-interactive mode with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin, patch.object(backend, 'iter_sql_rows') as rows: stdin.isatty.return_value = False with pytest.raises(SystemExit, match='--yes'): - list(backend.iter_records(query)) + list(backend.records(query)) rows.assert_not_called() backend.assume_yes = True with patch.object(backend, 'iter_sql_rows', return_value=iter([{'url': 'https://example.com/'}])) as rows: - assert list(backend.iter_records(query)) == [{'url': 'https://example.com/'}] + assert list(backend.records(query)) == [{'url': 'https://example.com/'}] assert rows.call_args.args == (sql,) @@ -295,7 +297,7 @@ def test_check_language(code, language): def test_languages_are_normalized_in_the_query(): - sql, _ = AthenaBackend(ATHENA).prepare(IndexQuery(hostnames=['example.com'], languages=['DEU'])) + sql = AthenaBackend(ATHENA).plan(IndexQuery(hostnames=['example.com'], languages=['DEU'])).sql assert "content_languages = 'deu'" in sql From ca39e47caa1f14d376246bfce7cf3068fb3b8572 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 12:50:12 +0200 Subject: [PATCH 42/91] refactor(index): AthenaLocation and AthenaClient AthenaLocation.parse() replaces the 4-tuple of parse_athena_location; AthenaClient (database, S3 output location, boto3 client) replaces the functions that passed the client around (run, rows, report_cost). --- cdx_toolkit/indexes/backends/athena.py | 290 +++++++++++++------------ tests/unit/test_athena.py | 28 ++- tests/unit/test_sql_indexes.py | 10 +- 3 files changed, 169 insertions(+), 159 deletions(-) diff --git a/cdx_toolkit/indexes/backends/athena.py b/cdx_toolkit/indexes/backends/athena.py index 9105d78..2dc34de 100644 --- a/cdx_toolkit/indexes/backends/athena.py +++ b/cdx_toolkit/indexes/backends/athena.py @@ -1,10 +1,11 @@ ''' Index backend for the columnar URL index queried with AWS Athena (boto3). ''' +from dataclasses import dataclass import logging import re import time -from typing import Iterable, Iterator, Optional, Tuple +from typing import Iterator, Optional from urllib.parse import parse_qs, urlparse from cdx_toolkit.aws import botocore_config @@ -19,123 +20,6 @@ ATHENA_USD_PER_TB = 5.0 -def make_client(region_name: Optional[str] = None): - import boto3 - - # retries: Common Crawl's recommended AWS settings, unless configured by the user (see cdx_toolkit.aws) - return boto3.client('athena', config=botocore_config(region_name=region_name, read_timeout=60)) - - -def iter_athena_rows(query: str, database: str, s3_output_location: str, region_name: Optional[str] = None, - max_wait_time: int = 300) -> Iterator[dict]: - """Run the query and yield the result rows as dicts (column name -> string value).""" - client = make_client(region_name) - query_execution_id = run_athena_query(client, query, database, s3_output_location, max_wait_time) - try: - yield from iter_result_rows(client, query_execution_id) - finally: - report_query_cost(client, query_execution_id) - - -def run_athena_query(client, query: str, database: str, s3_output_location: str, max_wait_time: int = 300) -> str: - """Start an Athena query and block until it completes; return the execution id. - - Raises if the query does not reach the SUCCEEDED state. If the wait is - interrupted (Ctrl-C / cancellation) or times out, the query is cancelled - server-side so we don't keep paying for a scan whose results we'll never read.""" - logger.info('Executing Athena query: %s', query) - - response = client.start_query_execution( - QueryString=query, - QueryExecutionContext={'Database': database}, - ResultConfiguration={'OutputLocation': s3_output_location}, - ) - - query_execution_id = response['QueryExecutionId'] - logger.info('Query execution started. ID: %s', query_execution_id) - - try: - status = _wait_for_query_completion(client, query_execution_id, max_wait_time) - except BaseException: - # Ctrl-C, asyncio cancellation, or timeout: stop the query server-side - # to bound the Athena scan cost, then propagate the original exception. - _stop_query(client, query_execution_id) - raise - - if status != 'SUCCEEDED': - raise Exception(f'Query failed with status: {status}') - - return query_execution_id - - -def _stop_query(client, query_execution_id: str) -> None: - """Best-effort cancellation of a running Athena query.""" - logger.warning('Cancelling Athena query %s ...', query_execution_id) - try: - client.stop_query_execution(QueryExecutionId=query_execution_id) - logger.warning('Athena query %s cancelled', query_execution_id) - except Exception as e: # pragma: no cover - best-effort cleanup - logger.warning('Failed to cancel Athena query %s: %r', query_execution_id, e) - - -def report_query_cost(client, query_execution_id: str) -> None: - """Log the bytes scanned and an estimated USD cost for a completed query.""" - try: - response = client.get_query_execution(QueryExecutionId=query_execution_id) - scanned = response['QueryExecution'].get('Statistics', {}).get('DataScannedInBytes') - except Exception as e: # pragma: no cover - best-effort reporting - logger.debug('unable to read Athena query statistics: %r', e) - return - - if scanned is None: - return - - gb = scanned / 1e9 - usd = scanned / 1e12 * ATHENA_USD_PER_TB - logger.info('Athena scanned %.2f GB, estimated cost ~$%.4f', gb, usd) - - -def _wait_for_query_completion(client, query_execution_id: str, max_wait_time: int) -> str: - """Wait for query to complete and return final status""" - start_time = time.time() - - while time.time() - start_time < max_wait_time: - response = client.get_query_execution(QueryExecutionId=query_execution_id) - - status = response['QueryExecution']['Status']['State'] - logger.info(f'Query status: {status}') - - if status in ['SUCCEEDED', 'FAILED', 'CANCELLED']: - if status == 'FAILED': - error_reason = response['QueryExecution']['Status'].get('StateChangeReason', 'Unknown error') - logger.info(f'Query failed: {error_reason}') - return status - - time.sleep(2) - - raise TimeoutError(f'Query did not complete within {max_wait_time} seconds') - - -def iter_result_rows(client, query_execution_id: str) -> Iterable[dict]: - """Retrieve query results as dicts (column name -> string value)""" - paginator = client.get_paginator('get_query_results') - page_iterator = paginator.paginate(QueryExecutionId=query_execution_id) - column_names = None - - for page in page_iterator: - rows = page['ResultSet']['Rows'] - - # Get column names from first page - if column_names is None and rows: - column_names = [col['VarCharValue'] for col in rows[0]['Data']] - rows = rows[1:] # Skip header row - - # Process data rows - for row in rows: - row_data = [cell.get('VarCharValue', None) for cell in row['Data']] - yield dict(zip(column_names, row_data)) - - class AthenaBackend(SqlBackend): """The columnar index queried with AWS Athena. @@ -154,29 +38,157 @@ class AthenaBackend(SqlBackend): def __init__(self, location: str, assume_yes: bool = False): super().__init__(location, assume_yes) - self.region, self.database, self.table, self.s3_output_location = parse_athena_location(location) + self.athena = AthenaLocation.parse(location) def from_clause(self, crawls): - return self.table, False + return self.athena.table, False def iter_sql_rows(self, sql): - return iter_athena_rows(sql, self.database, self.s3_output_location, region_name=self.region) - - -def parse_athena_location(location: str) -> Tuple[str, str, str, str]: - """athena://athena.us-east-1.amazonaws.com:443/ccindex?S3OutputLocation=s3://bucket/path/ - -> (region, database, table, S3 output location)""" - parsed = urlparse(location) - match = re.fullmatch(r'athena\.([a-z0-9-]+)\.amazonaws\.com', parsed.hostname or '') - if parsed.scheme != 'athena' or not match: - raise UsageError( - f'invalid Athena location {location!r}, expected ' - 'athena://athena..amazonaws.com/?S3OutputLocation=s3:////' + client = AthenaClient(self.athena.database, self.athena.s3_output_location, region_name=self.athena.region) + return client.iter_rows(sql) + + +@dataclass(frozen=True) +class AthenaLocation: + """Where an index is queried with Athena, and where Athena writes the results.""" + + region: str + database: str + table: str + s3_output_location: str + + @classmethod + def parse(cls, location: str) -> 'AthenaLocation': + """athena://athena.us-east-1.amazonaws.com:443/ccindex?S3OutputLocation=s3://bucket/path/[&table=ccindex]""" + parsed = urlparse(location) + match = re.fullmatch(r'athena\.([a-z0-9-]+)\.amazonaws\.com', parsed.hostname or '') + if parsed.scheme != 'athena' or not match: + raise UsageError( + f'invalid Athena location {location!r}, expected ' + 'athena://athena..amazonaws.com/?S3OutputLocation=s3:////' + ) + params = parse_qs(parsed.query) + s3_output_location = params.get('S3OutputLocation', [None])[0] + if not s3_output_location or not s3_output_location.startswith('s3://'): + raise UsageError(f'the Athena location {location!r} needs a S3OutputLocation=s3://... parameter') + database = parsed.path.strip('/') or 'ccindex' + table = params.get('table', ['ccindex'])[0] + return cls(match.group(1), database, table, s3_output_location) + + +def make_client(region_name: Optional[str] = None): + import boto3 + + # retries: Common Crawl's recommended AWS settings, unless configured by the user (see cdx_toolkit.aws) + return boto3.client('athena', config=botocore_config(region_name=region_name, read_timeout=60)) + + +class AthenaClient: + """Runs queries in an Athena database; Athena writes the results to `s3_output_location`.""" + + def __init__(self, database: str, s3_output_location: str, region_name: Optional[str] = None, client=None, + max_wait_time: int = 300): + self.database = database + self.s3_output_location = s3_output_location + self.client = client or make_client(region_name) + self.max_wait_time = max_wait_time + + def iter_rows(self, sql: str) -> Iterator[dict]: + """Run the query and yield the result rows as dicts (column name -> string value).""" + query_execution_id = self.run(sql) + try: + yield from self.rows(query_execution_id) + finally: + self.report_cost(query_execution_id) + + def run(self, sql: str) -> str: + """Start an Athena query and block until it completes; return the execution id. + + Raises if the query does not reach the SUCCEEDED state. If the wait is + interrupted (Ctrl-C / cancellation) or times out, the query is cancelled + server-side so we don't keep paying for a scan whose results we'll never read.""" + logger.info('Executing Athena query: %s', sql) + + response = self.client.start_query_execution( + QueryString=sql, + QueryExecutionContext={'Database': self.database}, + ResultConfiguration={'OutputLocation': self.s3_output_location}, ) - params = parse_qs(parsed.query) - s3_output_location = params.get('S3OutputLocation', [None])[0] - if not s3_output_location or not s3_output_location.startswith('s3://'): - raise UsageError(f'the Athena location {location!r} needs a S3OutputLocation=s3://... parameter') - database = parsed.path.strip('/') or 'ccindex' - table = params.get('table', ['ccindex'])[0] - return match.group(1), database, table, s3_output_location + + query_execution_id = response['QueryExecutionId'] + logger.info('Query execution started. ID: %s', query_execution_id) + + try: + status = self._wait_for_completion(query_execution_id) + except BaseException: + # Ctrl-C, asyncio cancellation, or timeout: stop the query server-side + # to bound the Athena scan cost, then propagate the original exception. + self._stop(query_execution_id) + raise + + if status != 'SUCCEEDED': + raise Exception(f'Query failed with status: {status}') + + return query_execution_id + + def rows(self, query_execution_id: str) -> Iterator[dict]: + """The results of a completed query as dicts (column name -> string value).""" + paginator = self.client.get_paginator('get_query_results') + column_names = None + + for page in paginator.paginate(QueryExecutionId=query_execution_id): + rows = page['ResultSet']['Rows'] + + # Get column names from first page + if column_names is None and rows: + column_names = [col['VarCharValue'] for col in rows[0]['Data']] + rows = rows[1:] # Skip header row + + for row in rows: + row_data = [cell.get('VarCharValue', None) for cell in row['Data']] + yield dict(zip(column_names, row_data)) + + def report_cost(self, query_execution_id: str) -> None: + """Log the bytes scanned and an estimated USD cost for a completed query.""" + try: + response = self.client.get_query_execution(QueryExecutionId=query_execution_id) + scanned = response['QueryExecution'].get('Statistics', {}).get('DataScannedInBytes') + except Exception as e: # pragma: no cover - best-effort reporting + logger.debug('unable to read Athena query statistics: %r', e) + return + + if scanned is None: + return + + gb = scanned / 1e9 + usd = scanned / 1e12 * ATHENA_USD_PER_TB + logger.info('Athena scanned %.2f GB, estimated cost ~$%.4f', gb, usd) + + def _wait_for_completion(self, query_execution_id: str) -> str: + """Wait for query to complete and return final status""" + start_time = time.time() + + while time.time() - start_time < self.max_wait_time: + response = self.client.get_query_execution(QueryExecutionId=query_execution_id) + + status = response['QueryExecution']['Status']['State'] + logger.info(f'Query status: {status}') + + if status in ['SUCCEEDED', 'FAILED', 'CANCELLED']: + if status == 'FAILED': + error_reason = response['QueryExecution']['Status'].get('StateChangeReason', 'Unknown error') + logger.info(f'Query failed: {error_reason}') + return status + + time.sleep(2) + + raise TimeoutError(f'Query did not complete within {self.max_wait_time} seconds') + + def _stop(self, query_execution_id: str) -> None: + """Best-effort cancellation of a running Athena query.""" + logger.warning('Cancelling Athena query %s ...', query_execution_id) + try: + self.client.stop_query_execution(QueryExecutionId=query_execution_id) + logger.warning('Athena query %s cancelled', query_execution_id) + except Exception as e: # pragma: no cover - best-effort cleanup + logger.warning('Failed to cancel Athena query %s: %r', query_execution_id, e) diff --git a/tests/unit/test_athena.py b/tests/unit/test_athena.py index e525c2b..a4da261 100644 --- a/tests/unit/test_athena.py +++ b/tests/unit/test_athena.py @@ -7,9 +7,7 @@ from cdx_toolkit.cli import main from cdx_toolkit.indexes import IndexQuery -from cdx_toolkit.indexes.backends.athena import ( - AthenaBackend, iter_athena_rows, make_client, report_query_cost, run_athena_query, -) +from cdx_toolkit.indexes.backends.athena import AthenaBackend, AthenaClient, make_client ATHENA = 'athena://athena.eu-west-1.amazonaws.com/mydb?S3OutputLocation=s3://bucket/athena/' COLUMNS = ['url', 'fetch_time', 'fetch_status', 'warc_filename', 'warc_record_offset', 'warc_record_length'] @@ -65,7 +63,7 @@ def client(): def test_rows_and_cost(client, caplog): with caplog.at_level(logging.INFO): - rows = list(iter_athena_rows('SELECT 1', 'ccindex', 's3://b/p/')) + rows = list(AthenaClient('ccindex', 's3://b/p/').iter_rows('SELECT 1')) assert rows == [dict(zip(COLUMNS, row)) for row in ROWS] assert client.started[0]['QueryExecutionContext'] == {'Database': 'ccindex'} assert client.started[0]['ResultConfiguration'] == {'OutputLocation': 's3://b/p/'} @@ -99,7 +97,7 @@ def test_athena_count(): def test_failed_query(caplog): fake = FakeAthenaClient(states=['FAILED'], reason='SYNTAX_ERROR: line 1:8') with caplog.at_level(logging.INFO), pytest.raises(Exception, match='Query failed with status: FAILED'): - run_athena_query(fake, 'SELECT', 'ccindex', 's3://b/p/') + AthenaClient('ccindex', 's3://b/p/', client=fake).run('SELECT') assert 'SYNTAX_ERROR' in caplog.text assert fake.stopped == [] @@ -107,13 +105,13 @@ def test_failed_query(caplog): def test_query_timeout_is_cancelled(): fake = FakeAthenaClient(states=['RUNNING']) with patch('cdx_toolkit.indexes.backends.athena.time.sleep'), pytest.raises(TimeoutError): - run_athena_query(fake, 'SELECT 1', 'ccindex', 's3://b/p/', max_wait_time=0) + AthenaClient('ccindex', 's3://b/p/', client=fake, max_wait_time=0).run('SELECT 1') assert fake.stopped == ['qid'] def test_cost_report_without_statistics(caplog): with caplog.at_level(logging.INFO): - report_query_cost(FakeAthenaClient(scanned=None), 'qid') + AthenaClient('ccindex', 's3://b/p/', client=FakeAthenaClient(scanned=None)).report_cost('qid') assert 'Athena scanned' not in caplog.text @@ -152,30 +150,30 @@ def stop_query_execution(self, QueryExecutionId): self.stopped.append(QueryExecutionId) -def test_run_athena_query_logs_sql(caplog): +def test_run_logs_sql(caplog): import logging client = _FakeAthenaClient() with caplog.at_level(logging.INFO): - run_athena_query(client, 'SELECT my_special_column FROM ccindex', 'ccindex', 's3://b/out/') + AthenaClient('ccindex', 's3://b/out/', client=client).run('SELECT my_special_column FROM ccindex') assert 'SELECT my_special_column FROM ccindex' in caplog.text -def test_run_athena_query_success_no_cancel(): +def test_run_success_no_cancel(): client = _FakeAthenaClient() - qid = run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') + qid = AthenaClient('ccindex', 's3://b/out/', client=client).run('SELECT 1') assert qid == 'qid-123' assert client.stopped == [] -def test_run_athena_query_cancels_on_interrupt(): +def test_run_cancels_on_interrupt(): client = _FakeAthenaClient(raise_on_wait=KeyboardInterrupt()) with pytest.raises(KeyboardInterrupt): - run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') + AthenaClient('ccindex', 's3://b/out/', client=client).run('SELECT 1') assert client.stopped == ['qid-123'] -def test_run_athena_query_cancels_on_timeout(): +def test_run_cancels_on_timeout(): client = _FakeAthenaClient(raise_on_wait=TimeoutError('too slow')) with pytest.raises(TimeoutError): - run_athena_query(client, 'SELECT 1', 'ccindex', 's3://b/out/') + AthenaClient('ccindex', 's3://b/out/', client=client).run('SELECT 1') assert client.stopped == ['qid-123'] diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index b963239..43ce19c 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -8,7 +8,7 @@ from cdx_toolkit.indexes import IndexQuery, UsageError, resolve_index from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend -from cdx_toolkit.indexes.backends.athena import AthenaBackend, parse_athena_location +from cdx_toolkit.indexes.backends.athena import AthenaBackend, AthenaLocation from cdx_toolkit.indexes.backends.duckdb import DuckDbBackend from cdx_toolkit.indexes.backends.sql import normalize_row from cdx_toolkit.indexes.backends.sql_builder import check_language, query_conditions @@ -53,11 +53,11 @@ def test_resolve_index_errors(location, index_format, error): resolve_index(location, index_format) -def test_parse_athena_location(): - assert parse_athena_location(ATHENA) == ('us-east-1', 'ccindex', 'ccindex', 's3://my-bucket/athena/') - assert parse_athena_location( +def test_athena_location(): + assert AthenaLocation.parse(ATHENA) == AthenaLocation('us-east-1', 'ccindex', 'ccindex', 's3://my-bucket/athena/') + assert AthenaLocation.parse( 'athena://athena.eu-west-1.amazonaws.com/mydb?S3OutputLocation=s3://b/p/&table=urls' - ) == ('eu-west-1', 'mydb', 'urls', 's3://b/p/') + ) == AthenaLocation('eu-west-1', 'mydb', 'urls', 's3://b/p/') def test_normalize_row(): From 11fcacf07e939170adfca4330c453cee89a7ad23 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 15:07:59 +0200 Subject: [PATCH 43/91] refactor(repackage): readers/ and writers/ packages, jobs.py, urls.py, retries.py, warc_records.py Moves and imports only: range jobs and the records source in jobs.py, the HTTP/HF readers in readers/, the local/S3 output files and the output series helpers in writers/, URL schemes in urls.py, the retry helper in retries.py, warcinfo/metadata record bytes in warc_records.py, and the pipeline (WARCFilter) in pipeline.py. --- cdx_toolkit/cli/repackage.py | 10 +- .../repackage/{data_classes.py => jobs.py} | 131 ++++++++--------- cdx_toolkit/repackage/multiprocess.py | 9 +- .../repackage/{warc_filter.py => pipeline.py} | 70 ++++----- cdx_toolkit/repackage/readers/__init__.py | 35 +++++ .../repackage/{hf_utils.py => readers/hf.py} | 9 +- .../{http_range.py => readers/http.py} | 34 +++++ cdx_toolkit/repackage/records_source.py | 61 -------- .../repackage/{s3_utils.py => retries.py} | 17 +-- cdx_toolkit/repackage/urls.py | 26 ++++ cdx_toolkit/repackage/warc_records.py | 71 ++++++++++ cdx_toolkit/repackage/warc_utils.py | 134 ------------------ cdx_toolkit/repackage/writers/__init__.py | 34 +++++ .../{local_writer.py => writers/local.py} | 0 .../repackage/{s3_writer.py => writers/s3.py} | 4 +- cdx_toolkit/repackage/writers/series.py | 41 ++++++ tests/fake_warc_server.py | 2 +- tests/repackage/test_data_classes.py | 4 +- .../{test_hf_utils.py => test_hf_reader.py} | 8 +- .../{test_records_source.py => test_jobs.py} | 2 +- ...t_local_writer.py => test_local_output.py} | 2 +- .../{test_warc_filter.py => test_pipeline.py} | 9 +- tests/repackage/test_producer.py | 4 +- .../{test_s3_utils.py => test_retries.py} | 7 +- .../{test_s3_writer.py => test_s3_output.py} | 2 +- tests/repackage/test_warc_filter_pipeline.py | 40 +++--- ...est_warc_utils.py => test_warc_records.py} | 3 +- tests/unit/test_aws.py | 4 +- tests/unit/test_cli.py | 22 +-- tests/unit/test_sql_indexes.py | 2 +- tests/unit/test_usage_errors.py | 2 +- 31 files changed, 410 insertions(+), 389 deletions(-) rename cdx_toolkit/repackage/{data_classes.py => jobs.py} (60%) rename cdx_toolkit/repackage/{warc_filter.py => pipeline.py} (93%) create mode 100644 cdx_toolkit/repackage/readers/__init__.py rename cdx_toolkit/repackage/{hf_utils.py => readers/hf.py} (97%) rename cdx_toolkit/repackage/{http_range.py => readers/http.py} (55%) delete mode 100644 cdx_toolkit/repackage/records_source.py rename cdx_toolkit/repackage/{s3_utils.py => retries.py} (82%) create mode 100644 cdx_toolkit/repackage/urls.py create mode 100644 cdx_toolkit/repackage/warc_records.py delete mode 100644 cdx_toolkit/repackage/warc_utils.py create mode 100644 cdx_toolkit/repackage/writers/__init__.py rename cdx_toolkit/repackage/{local_writer.py => writers/local.py} (100%) rename cdx_toolkit/repackage/{s3_writer.py => writers/s3.py} (98%) create mode 100644 cdx_toolkit/repackage/writers/series.py rename tests/repackage/{test_hf_utils.py => test_hf_reader.py} (97%) rename tests/repackage/{test_records_source.py => test_jobs.py} (93%) rename tests/repackage/{test_local_writer.py => test_local_output.py} (97%) rename tests/repackage/{test_warc_filter.py => test_pipeline.py} (85%) rename tests/repackage/{test_s3_utils.py => test_retries.py} (97%) rename tests/repackage/{test_s3_writer.py => test_s3_output.py} (98%) rename tests/repackage/{test_warc_utils.py => test_warc_records.py} (91%) diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index 99a80da..82e1d80 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -172,7 +172,7 @@ def run(args: Namespace, cmdline: str): def check_output_is_new(prefix: str, subprefix: Optional[str]) -> None: """Every run numbers its files from the start: never overwrite (or mix with) the files of an earlier run, with or without --wb.""" - from cdx_toolkit.repackage.warc_utils import existing_warc_files + from cdx_toolkit.repackage.writers.series import existing_warc_files existing = existing_warc_files(prefix, subprefix) if existing: @@ -186,8 +186,8 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): """Copy the WARC records byte-for-byte with the async WARC fetcher (optionally multi-process).""" import fsspec - from cdx_toolkit.repackage.hf_utils import is_hf_url - from cdx_toolkit.repackage.records_source import MissingWarcLocation, RecordsSource + from cdx_toolkit.repackage.urls import is_hf_url + from cdx_toolkit.repackage.jobs import MissingWarcLocation, RecordsSource try: source = RecordsSource(records, args.warc_download_prefix, sort=not args.no_sort) @@ -213,7 +213,7 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): max_file_size=args.size, ) - from cdx_toolkit.repackage.warc_filter import WARCFilter + from cdx_toolkit.repackage.pipeline import WARCFilter return WARCFilter( source=source, @@ -239,7 +239,7 @@ def repackage_from_wayback(args: Namespace, records, info: dict): if used: raise UsageError(f'{option} cannot be used with --wb') - from cdx_toolkit.repackage.data_classes import RepackageResult + from cdx_toolkit.repackage.jobs import RepackageResult wb = WAYBACK_NAMES.get(args.wb, args.wb) requested = written = 0 diff --git a/cdx_toolkit/repackage/data_classes.py b/cdx_toolkit/repackage/jobs.py similarity index 60% rename from cdx_toolkit/repackage/data_classes.py rename to cdx_toolkit/repackage/jobs.py index a1d720c..40dde84 100644 --- a/cdx_toolkit/repackage/data_classes.py +++ b/cdx_toolkit/repackage/jobs.py @@ -1,75 +1,18 @@ -import asyncio +''' +The captures to repackage: index records (as written by `cdxt index`) -> range jobs, and the result of a run. +''' +import logging import threading -import time from dataclasses import dataclass - -from cdx_toolkit.repackage.s3_utils import RETRYABLE_ERRORS, is_s3_url, parse_s3_uri, with_retries -from cdx_toolkit.repackage.hf_utils import is_hf_url -from typing import Optional, Tuple +from typing import Iterable, Iterator, Mapping, Optional, Tuple from cdx_toolkit.myrequests import myrequests_get -from cdx_toolkit.repackage.http_range import RangeError, check_range_response - - -async def run_in_daemon_thread(func, *args, **kwargs): - """Run a blocking function in a daemon thread of its own (like asyncio.to_thread). - - Unlike asyncio.to_thread, nothing waits for a cancelled call: not the cancelled task, not asyncio.run() (which - waits for the threads of the default executor) and not the interpreter at exit.""" - loop = asyncio.get_running_loop() - future = loop.create_future() - - def set_result(result, exception): - if not future.cancelled(): - if exception is not None: - future.set_exception(exception) - else: - future.set_result(result) - - def run(): - try: - result, exception = func(*args, **kwargs), None - except BaseException as e: - result, exception = None, e - try: - loop.call_soon_threadsafe(set_result, result, exception) - except RuntimeError: - pass # the event loop is closed: nobody waits for the result anymore - - threading.Thread(target=run, daemon=True).start() - return await future - - -@dataclass -class ThroughputTracker: - """Track throughput metrics for fetchers and consumers.""" - - start_time: float = 0.0 - total_bytes: int = 0 - total_requests: int = 0 - total_records: int = 0 - - def start(self): - self.start_time = time.time() - - def add(self, bytes_count: int = 0, records_count: int = 0, requests_count: int = 1): - self.total_bytes += bytes_count - self.total_requests += requests_count - self.total_records += records_count - - def get_stats(self) -> dict: - elapsed = time.time() - self.start_time - - return { - 'elapsed': elapsed, - 'total_bytes': self.total_bytes, - 'total_requests': self.total_requests, - 'total_records': self.total_records, - 'bytes_per_sec': self.total_bytes / elapsed if elapsed > 0 else 0, - 'mb_per_sec': (self.total_bytes / elapsed) / (1024 * 1024) if elapsed > 0 else 0, - 'requests_per_sec': self.total_requests / elapsed if elapsed > 0 else 0, - 'records_per_sec': self.total_records / elapsed if elapsed > 0 else 0, - } +from cdx_toolkit.repackage.readers.http import RangeError, check_range_response, run_in_daemon_thread +from cdx_toolkit.repackage.retries import RETRYABLE_ERRORS, with_retries +from cdx_toolkit.repackage.urls import is_hf_url, is_s3_url, parse_s3_uri + + +logger = logging.getLogger(__name__) @dataclass(frozen=True) @@ -183,3 +126,55 @@ class RangePayload: job: RangeJob data: bytes + + +WARC_LOCATION_FIELDS = ('filename', 'offset', 'length') + + +class MissingWarcLocation(ValueError): + """A record has no WARC location (filename, offset, length), e.g. an Internet Archive capture.""" + + +def join_warc_url(prefix: Optional[str], warc_filename: str) -> str: + """Join a download prefix with a warc_filename robustly. + + - if warc_filename is already absolute (contains '://'), return it unchanged + (supports custom queries / self-contained index files whose value is a full URL); + - if prefix is empty/None, return warc_filename unchanged; + - otherwise join with exactly one '/' (no double slash, no missing slash).""" + if '://' in warc_filename: + return warc_filename + if not prefix: + return warc_filename + return prefix.rstrip('/') + '/' + warc_filename.lstrip('/') + + +class RecordsSource: + """RangeJobs from index records (as written by `cdxt index`) with filename, offset and length fields. + + With `sort` (the default), jobs are sorted by (WARC file, record offset) for better + range-read locality. The records are then read (and validated) when the source is + created, and kept in memory; pass sort=False to stream them in input order.""" + + def __init__(self, records: Iterable[Mapping], warc_download_prefix: Optional[str], sort: bool = True): + self.warc_download_prefix = warc_download_prefix + self.jobs = (self.to_range_job(record) for record in records) + if sort: + self.jobs = sorted(self.jobs, key=lambda j: (j.filename or j.url, j.offset)) + + def iter_range_jobs(self) -> Iterator[RangeJob]: + """Yield the range jobs; `url` is authoritative for fetching, `filename` is the filename of the record.""" + yield from self.jobs + + def to_range_job(self, record: Mapping) -> RangeJob: + if not all(record.get(field) not in (None, '') for field in WARC_LOCATION_FIELDS): + raise MissingWarcLocation( + f'capture {record.get("url")} has no WARC location (filename, offset, length); ' + 'use --wb to fetch it from a wayback machine' + ) + return RangeJob( + url=join_warc_url(self.warc_download_prefix, record['filename']), + offset=int(record['offset']), + length=int(record['length']), + filename=record['filename'], + ) diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index 12dbc01..bf54c15 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -37,11 +37,10 @@ from cdx_toolkit.aws import configure_fsspec_s3 from cdx_toolkit.record_io import RecordWriter, read_records -from cdx_toolkit.repackage.data_classes import RepackageResult -from cdx_toolkit.repackage.records_source import RecordsSource -from cdx_toolkit.repackage.warc_filter import WARCFilter -from cdx_toolkit.repackage.warc_utils import generate_warc_filename -from cdx_toolkit.repackage.s3_utils import is_s3_url, parse_s3_uri +from cdx_toolkit.repackage.jobs import RecordsSource, RepackageResult +from cdx_toolkit.repackage.pipeline import WARCFilter +from cdx_toolkit.repackage.urls import is_s3_url, parse_s3_uri +from cdx_toolkit.repackage.writers.series import generate_warc_filename logger = logging.getLogger(__name__) diff --git a/cdx_toolkit/repackage/warc_filter.py b/cdx_toolkit/repackage/pipeline.py similarity index 93% rename from cdx_toolkit/repackage/warc_filter.py rename to cdx_toolkit/repackage/pipeline.py index 48c2bbc..a10d5f2 100644 --- a/cdx_toolkit/repackage/warc_filter.py +++ b/cdx_toolkit/repackage/pipeline.py @@ -5,18 +5,19 @@ import os import statistics import threading +import time +from dataclasses import dataclass from typing import Callable, List, Optional, Dict from cdx_toolkit.aws import botocore_config -from cdx_toolkit.repackage.s3_utils import ( - is_s3_url, -) -from cdx_toolkit.repackage.hf_utils import make_hf_reader -from cdx_toolkit.repackage.data_classes import RangeJob, RangePayload, RepackageResult, ThroughputTracker -from cdx_toolkit.repackage.warc_utils import ( - get_bytes_from_warc_record, get_metadata_record_from_path, new_warc_writer, warcinfo_record_bytes, +from cdx_toolkit.repackage.jobs import RangeJob, RangePayload, RepackageResult +from cdx_toolkit.repackage.readers import Readers +from cdx_toolkit.repackage.urls import is_s3_url +from cdx_toolkit.repackage.warc_records import ( + get_bytes_from_warc_record, get_metadata_record_from_path, warcinfo_record_bytes, ) +from cdx_toolkit.repackage.writers import new_warc_writer _STOP = object() @@ -24,31 +25,36 @@ logger = logging.getLogger(__name__) -class Readers: - """The readers of WARC ranges, by the URL scheme of a range job, each created when it is first needed: - an S3 client for s3:// and the HF reader for hf:// (http(s):// needs none). They are closed with `stack`.""" - - def __init__(self, stack: contextlib.AsyncExitStack, s3_config_factory: Callable): - self.stack = stack - self.s3_config_factory = s3_config_factory - self._s3 = None - self._hf = None - self._lock = asyncio.Lock() - - async def s3(self): - async with self._lock: - if self._s3 is None: - import aioboto3 - - client = aioboto3.Session().client('s3', config=self.s3_config_factory()) - self._s3 = await self.stack.enter_async_context(client) - return self._s3 - - async def hf(self): - async with self._lock: - if self._hf is None: - self._hf = await self.stack.enter_async_context(make_hf_reader()) - return self._hf +@dataclass +class ThroughputTracker: + """Track throughput metrics for fetchers and consumers.""" + + start_time: float = 0.0 + total_bytes: int = 0 + total_requests: int = 0 + total_records: int = 0 + + def start(self): + self.start_time = time.time() + + def add(self, bytes_count: int = 0, records_count: int = 0, requests_count: int = 1): + self.total_bytes += bytes_count + self.total_requests += requests_count + self.total_records += records_count + + def get_stats(self) -> dict: + elapsed = time.time() - self.start_time + + return { + 'elapsed': elapsed, + 'total_bytes': self.total_bytes, + 'total_requests': self.total_requests, + 'total_records': self.total_records, + 'bytes_per_sec': self.total_bytes / elapsed if elapsed > 0 else 0, + 'mb_per_sec': (self.total_bytes / elapsed) / (1024 * 1024) if elapsed > 0 else 0, + 'requests_per_sec': self.total_requests / elapsed if elapsed > 0 else 0, + 'records_per_sec': self.total_records / elapsed if elapsed > 0 else 0, + } class WARCFilter: diff --git a/cdx_toolkit/repackage/readers/__init__.py b/cdx_toolkit/repackage/readers/__init__.py new file mode 100644 index 0000000..14a56e8 --- /dev/null +++ b/cdx_toolkit/repackage/readers/__init__.py @@ -0,0 +1,35 @@ +''' +The readers of WARC ranges, by URL scheme: HTTP(S) (http.py), S3 (aioboto3) and Hugging Face buckets (hf.py). +''' +import asyncio +import contextlib +from typing import Callable + +from cdx_toolkit.repackage.readers.hf import make_hf_reader + + +class Readers: + """The readers of WARC ranges, by the URL scheme of a range job, each created when it is first needed: + an S3 client for s3:// and the HF reader for hf:// (http(s):// needs none). They are closed with `stack`.""" + + def __init__(self, stack: contextlib.AsyncExitStack, s3_config_factory: Callable): + self.stack = stack + self.s3_config_factory = s3_config_factory + self._s3 = None + self._hf = None + self._lock = asyncio.Lock() + + async def s3(self): + async with self._lock: + if self._s3 is None: + import aioboto3 + + client = aioboto3.Session().client('s3', config=self.s3_config_factory()) + self._s3 = await self.stack.enter_async_context(client) + return self._s3 + + async def hf(self): + async with self._lock: + if self._hf is None: + self._hf = await self.stack.enter_async_context(make_hf_reader()) + return self._hf diff --git a/cdx_toolkit/repackage/hf_utils.py b/cdx_toolkit/repackage/readers/hf.py similarity index 97% rename from cdx_toolkit/repackage/hf_utils.py rename to cdx_toolkit/repackage/readers/hf.py index 71517f7..46ab3d2 100644 --- a/cdx_toolkit/repackage/hf_utils.py +++ b/cdx_toolkit/repackage/readers/hf.py @@ -23,19 +23,14 @@ from typing import Tuple from urllib.parse import parse_qs, urlparse -from cdx_toolkit.repackage.http_range import check_range_response +from cdx_toolkit.repackage.readers.http import check_range_response +from cdx_toolkit.repackage.urls import HF_SCHEME, is_hf_url logger = logging.getLogger(__name__) -HF_SCHEME = 'hf://' HF_ENDPOINT = 'https://huggingface.co' -def is_hf_url(url) -> bool: - """True for an ``hf://`` URL (e.g. an hf:// WARC download prefix or job URL).""" - return bool(url) and url.startswith(HF_SCHEME) - - def hf_fs_path(url: str) -> str: """``hf://buckets/ns/name/key`` -> ``buckets/ns/name/key`` (HfFileSystem path).""" if not is_hf_url(url): diff --git a/cdx_toolkit/repackage/http_range.py b/cdx_toolkit/repackage/readers/http.py similarity index 55% rename from cdx_toolkit/repackage/http_range.py rename to cdx_toolkit/repackage/readers/http.py index 9017a2d..0a8cd8d 100644 --- a/cdx_toolkit/repackage/http_range.py +++ b/cdx_toolkit/repackage/readers/http.py @@ -1,4 +1,9 @@ +''' +Range reads over HTTP(S). +''' +import asyncio import re +import threading from typing import Optional @@ -25,3 +30,32 @@ def check_range_response(status: int, content_range: Optional[str], body_length: raise RangeError(f'{url}: unexpected status {status} for the range request for bytes {offset}-{end}') if body_length != length: raise RangeError(f'{url}: requested {length} bytes from offset {offset}, got {body_length}') + + +async def run_in_daemon_thread(func, *args, **kwargs): + """Run a blocking function in a daemon thread of its own (like asyncio.to_thread). + + Unlike asyncio.to_thread, nothing waits for a cancelled call: not the cancelled task, not asyncio.run() (which + waits for the threads of the default executor) and not the interpreter at exit.""" + loop = asyncio.get_running_loop() + future = loop.create_future() + + def set_result(result, exception): + if not future.cancelled(): + if exception is not None: + future.set_exception(exception) + else: + future.set_result(result) + + def run(): + try: + result, exception = func(*args, **kwargs), None + except BaseException as e: + result, exception = None, e + try: + loop.call_soon_threadsafe(set_result, result, exception) + except RuntimeError: + pass # the event loop is closed: nobody waits for the result anymore + + threading.Thread(target=run, daemon=True).start() + return await future diff --git a/cdx_toolkit/repackage/records_source.py b/cdx_toolkit/repackage/records_source.py deleted file mode 100644 index 8dbe821..0000000 --- a/cdx_toolkit/repackage/records_source.py +++ /dev/null @@ -1,61 +0,0 @@ -''' -The captures to repackage: index records (as written by `cdxt index`) -> range jobs for the WARC fetcher. -''' -import logging -from typing import Iterable, Iterator, Mapping, Optional - -from cdx_toolkit.repackage.data_classes import RangeJob - - -logger = logging.getLogger(__name__) - -WARC_LOCATION_FIELDS = ('filename', 'offset', 'length') - - -class MissingWarcLocation(ValueError): - """A record has no WARC location (filename, offset, length), e.g. an Internet Archive capture.""" - - -def join_warc_url(prefix: Optional[str], warc_filename: str) -> str: - """Join a download prefix with a warc_filename robustly. - - - if warc_filename is already absolute (contains '://'), return it unchanged - (supports custom queries / self-contained index files whose value is a full URL); - - if prefix is empty/None, return warc_filename unchanged; - - otherwise join with exactly one '/' (no double slash, no missing slash).""" - if '://' in warc_filename: - return warc_filename - if not prefix: - return warc_filename - return prefix.rstrip('/') + '/' + warc_filename.lstrip('/') - - -class RecordsSource: - """RangeJobs from index records (as written by `cdxt index`) with filename, offset and length fields. - - With `sort` (the default), jobs are sorted by (WARC file, record offset) for better - range-read locality. The records are then read (and validated) when the source is - created, and kept in memory; pass sort=False to stream them in input order.""" - - def __init__(self, records: Iterable[Mapping], warc_download_prefix: Optional[str], sort: bool = True): - self.warc_download_prefix = warc_download_prefix - self.jobs = (self.to_range_job(record) for record in records) - if sort: - self.jobs = sorted(self.jobs, key=lambda j: (j.filename or j.url, j.offset)) - - def iter_range_jobs(self) -> Iterator[RangeJob]: - """Yield the range jobs; `url` is authoritative for fetching, `filename` is the filename of the record.""" - yield from self.jobs - - def to_range_job(self, record: Mapping) -> RangeJob: - if not all(record.get(field) not in (None, '') for field in WARC_LOCATION_FIELDS): - raise MissingWarcLocation( - f'capture {record.get("url")} has no WARC location (filename, offset, length); ' - 'use --wb to fetch it from a wayback machine' - ) - return RangeJob( - url=join_warc_url(self.warc_download_prefix, record['filename']), - offset=int(record['offset']), - length=int(record['length']), - filename=record['filename'], - ) diff --git a/cdx_toolkit/repackage/s3_utils.py b/cdx_toolkit/repackage/retries.py similarity index 82% rename from cdx_toolkit/repackage/s3_utils.py rename to cdx_toolkit/repackage/retries.py index cddb3a5..caaaf16 100644 --- a/cdx_toolkit/repackage/s3_utils.py +++ b/cdx_toolkit/repackage/retries.py @@ -1,6 +1,6 @@ import asyncio import logging -from typing import Optional, Tuple +from typing import Tuple from os import urandom try: @@ -30,21 +30,6 @@ class EndpointConnectionError(Exception): logger = logging.getLogger(__name__) -def parse_s3_uri(uri: str) -> Tuple[str, str]: - """Parse a S3 URI and return bucket and prefix.""" - if not uri.startswith('s3://'): - raise ValueError(f'Not an S3 URI: {uri}') - rest = uri[5:] - i = rest.find('/') - if i <= 0 or i == len(rest) - 1: - raise ValueError(f'Malformed S3 URI: {uri}') - return rest[:i], rest[i+1:] - - -def is_s3_url(url: Optional[str]) -> bool: - return bool(url) and url.startswith('s3:/') - - async def with_retries(coro_factory, *, op_name: str, max_attempts: int, base_backoff_seconds: float, retry_on: Tuple[type, ...] = RETRYABLE_ERRORS): """Perform operation with retries and backoff.""" diff --git a/cdx_toolkit/repackage/urls.py b/cdx_toolkit/repackage/urls.py new file mode 100644 index 0000000..8a5b719 --- /dev/null +++ b/cdx_toolkit/repackage/urls.py @@ -0,0 +1,26 @@ +''' +The URL schemes of WARC files and outputs that need their own readers or writers: s3:// and hf://. +''' +from typing import Optional, Tuple + +HF_SCHEME = 'hf://' + + +def parse_s3_uri(uri: str) -> Tuple[str, str]: + """Parse a S3 URI and return bucket and prefix.""" + if not uri.startswith('s3://'): + raise ValueError(f'Not an S3 URI: {uri}') + rest = uri[5:] + i = rest.find('/') + if i <= 0 or i == len(rest) - 1: + raise ValueError(f'Malformed S3 URI: {uri}') + return rest[:i], rest[i+1:] + + +def is_s3_url(url: Optional[str]) -> bool: + return bool(url) and url.startswith('s3:/') + + +def is_hf_url(url) -> bool: + """True for an ``hf://`` URL (e.g. an hf:// WARC download prefix or job URL).""" + return bool(url) and url.startswith(HF_SCHEME) diff --git a/cdx_toolkit/repackage/warc_records.py b/cdx_toolkit/repackage/warc_records.py new file mode 100644 index 0000000..9107d95 --- /dev/null +++ b/cdx_toolkit/repackage/warc_records.py @@ -0,0 +1,71 @@ +''' +The bytes of the WARC records that cdxt repackage writes itself: warcinfo and metadata records. +''' +from io import BytesIO +import logging +import mimetypes +from pathlib import Path +from typing import Dict, Tuple, Union + +import fsspec +from warcio import WARCWriter +from warcio.recordbuilder import RecordBuilder +from warcio.recordloader import ArcWarcRecord + +logger = logging.getLogger(__name__) + + +def get_bytes_from_warc_record( + record, + warc_version: str = '1.0', + gzip: bool = False, +): + """Get byte representation of WARC record.""" + buffer = BytesIO() + warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) + warc_writer.write_record(record) + + return buffer.getvalue() + + +def get_metadata_record_from_path( + file_path: Union[str, Path], + warcinfo_id: str, +) -> ArcWarcRecord: + """Build WARC metadata record for file path. + + The `Content-Type` header is guessed. + """ + # Cast to string + file_path = str(file_path) + + with fsspec.open(file_path, 'rb') as f: + file_bytes = BytesIO(f.read()) + + warc_content_type = mimetypes.guess_type(file_path)[0] + warc_headers_dict = {} + + # Set WARC-Warcinfo-ID + warc_headers_dict['WARC-Warcinfo-ID'] = warcinfo_id + + rb = RecordBuilder() + record = rb.create_warc_record( + uri=None, + record_type='metadata', + payload=file_bytes, + http_headers=None, + warc_content_type=warc_content_type, + warc_headers_dict=warc_headers_dict, + ) + + return record + + +def warcinfo_record_bytes(warc_file_name: str, writer_info: Dict, warc_version: str, gzip: bool) -> Tuple[bytes, str]: + """The warcinfo record of a WARC file (its `filename` field names the file) and its WARC-Record-ID (the + WARC-Warcinfo-ID of the file's other records).""" + buffer = BytesIO() + warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) + warcinfo = warc_writer.create_warcinfo_record(filename=warc_file_name, info=writer_info) + warc_writer.write_record(warcinfo) + return buffer.getvalue(), warcinfo.rec_headers.get('WARC-Record-ID') diff --git a/cdx_toolkit/repackage/warc_utils.py b/cdx_toolkit/repackage/warc_utils.py deleted file mode 100644 index 486a32c..0000000 --- a/cdx_toolkit/repackage/warc_utils.py +++ /dev/null @@ -1,134 +0,0 @@ -from io import BytesIO -import glob -import logging -import os -import posixpath -import re -from pathlib import Path -import fsspec -from warcio.recordloader import ArcWarcRecord -from warcio import WARCWriter - -from warcio.recordbuilder import RecordBuilder - -from typing import Dict, List, Optional, Tuple, Union - -import mimetypes - -from cdx_toolkit.repackage.s3_utils import is_s3_url, parse_s3_uri -from cdx_toolkit.repackage.local_writer import LocalFileWriter -from cdx_toolkit.repackage.s3_writer import S3ShardWriter - -logger = logging.getLogger(__name__) - - -def get_bytes_from_warc_record( - record, - warc_version: str = '1.0', - gzip: bool = False, -): - """Get byte representation of WARC record.""" - buffer = BytesIO() - warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) - warc_writer.write_record(record) - - return buffer.getvalue() - - -def get_metadata_record_from_path( - file_path: Union[str, Path], - warcinfo_id: str, -) -> ArcWarcRecord: - """Build WARC metadata record for file path. - - The `Content-Type` header is guessed. - """ - # Cast to string - file_path = str(file_path) - - with fsspec.open(file_path, 'rb') as f: - file_bytes = BytesIO(f.read()) - - warc_content_type = mimetypes.guess_type(file_path)[0] - warc_headers_dict = {} - - # Set WARC-Warcinfo-ID - warc_headers_dict['WARC-Warcinfo-ID'] = warcinfo_id - - rb = RecordBuilder() - record = rb.create_warc_record( - uri=None, - record_type='metadata', - payload=file_bytes, - http_headers=None, - warc_content_type=warc_content_type, - warc_headers_dict=warc_headers_dict, - ) - - return record - - -def generate_warc_filename( - dest_prefix: str, - sequence: int, - writer_subprefix: Optional[str] = None, - gzip: bool = False, -) -> str: - """Generate a WARC file name from a prefix (can be a full path), an optional subprefix - (used to distinguish per-process shards), and a sequence index (file-rotation counter).""" - file_name = dest_prefix + '-' - if writer_subprefix is not None: - file_name += writer_subprefix + '-' - file_name += f'{sequence:03d}.warc' # TODO default warc command uses ".extracted.warc" - if gzip: - file_name += '.gz' - - return file_name - - -def existing_warc_files(prefix_path: str, writer_subprefix: Optional[str] = None, gzip: bool = True) -> List[str]: - """The files of the series [-]-NNN.warc[.gz] (async WARC fetcher) or - [-]-NNNNNN.extracted.warc[.gz] (--wb) that already exist (local, s3://, hf://, ...).""" - fs, fs_prefix = fsspec.url_to_fs(prefix_path) - name = re.escape(posixpath.basename(fs_prefix)) + '-' - if writer_subprefix is not None: - name += re.escape(writer_subprefix) + '-' - name += r'\d{3,}(\.extracted)?\.warc' + (r'\.gz' if gzip else '') - return sorted( - fs.unstrip_protocol(path) for path in fs.glob(glob.escape(fs_prefix) + '-*') - if re.fullmatch(name, posixpath.basename(path)) - ) - - -def new_warc_writer( - sequence: int, - output_path_prefix: str, - max_attempts: int, - base_backoff_seconds: float, - min_part_size: int, - writer_subprefix: Optional[str] = None, - gzip: bool = False, - s3_client=None, -) -> Tuple[Union[S3ShardWriter, LocalFileWriter], str]: - """A new (not started) writer of a WARC file, local or S3, and the file's name (for the warcinfo record). - - Both writers have the same life cycle: start(), write()..., then close(), which completes the file, or - abort(), which removes it (or aborts its upload); a writer that cannot close aborts itself.""" - if is_s3_url(output_path_prefix): - dest_bucket, dest_prefix = parse_s3_uri(output_path_prefix) - key = generate_warc_filename(dest_prefix, sequence=sequence, writer_subprefix=writer_subprefix, gzip=gzip) - writer = S3ShardWriter(s3_client, key, dest_bucket, min_part_size, max_attempts, base_backoff_seconds) - return writer, key.split('/')[-1] - - path = generate_warc_filename(output_path_prefix, sequence=sequence, writer_subprefix=writer_subprefix, gzip=gzip) - return LocalFileWriter(file_path=path), os.path.basename(path) # with backslashes on Windows - - -def warcinfo_record_bytes(warc_file_name: str, writer_info: Dict, warc_version: str, gzip: bool) -> Tuple[bytes, str]: - """The warcinfo record of a WARC file (its `filename` field names the file) and its WARC-Record-ID (the - WARC-Warcinfo-ID of the file's other records).""" - buffer = BytesIO() - warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) - warcinfo = warc_writer.create_warcinfo_record(filename=warc_file_name, info=writer_info) - warc_writer.write_record(warcinfo) - return buffer.getvalue(), warcinfo.rec_headers.get('WARC-Record-ID') diff --git a/cdx_toolkit/repackage/writers/__init__.py b/cdx_toolkit/repackage/writers/__init__.py new file mode 100644 index 0000000..f23ef98 --- /dev/null +++ b/cdx_toolkit/repackage/writers/__init__.py @@ -0,0 +1,34 @@ +''' +The writers of output files: a local file (local.py) or an S3 object written with a multipart upload (s3.py). +''' +import os +from typing import Optional, Tuple, Union + +from cdx_toolkit.repackage.urls import is_s3_url, parse_s3_uri +from cdx_toolkit.repackage.writers.local import LocalFileWriter +from cdx_toolkit.repackage.writers.s3 import S3ShardWriter +from cdx_toolkit.repackage.writers.series import generate_warc_filename + + +def new_warc_writer( + sequence: int, + output_path_prefix: str, + max_attempts: int, + base_backoff_seconds: float, + min_part_size: int, + writer_subprefix: Optional[str] = None, + gzip: bool = False, + s3_client=None, +) -> Tuple[Union[S3ShardWriter, LocalFileWriter], str]: + """A new (not started) writer of a WARC file, local or S3, and the file's name (for the warcinfo record). + + Both writers have the same life cycle: start(), write()..., then close(), which completes the file, or + abort(), which removes it (or aborts its upload); a writer that cannot close aborts itself.""" + if is_s3_url(output_path_prefix): + dest_bucket, dest_prefix = parse_s3_uri(output_path_prefix) + key = generate_warc_filename(dest_prefix, sequence=sequence, writer_subprefix=writer_subprefix, gzip=gzip) + writer = S3ShardWriter(s3_client, key, dest_bucket, min_part_size, max_attempts, base_backoff_seconds) + return writer, key.split('/')[-1] + + path = generate_warc_filename(output_path_prefix, sequence=sequence, writer_subprefix=writer_subprefix, gzip=gzip) + return LocalFileWriter(file_path=path), os.path.basename(path) # with backslashes on Windows diff --git a/cdx_toolkit/repackage/local_writer.py b/cdx_toolkit/repackage/writers/local.py similarity index 100% rename from cdx_toolkit/repackage/local_writer.py rename to cdx_toolkit/repackage/writers/local.py diff --git a/cdx_toolkit/repackage/s3_writer.py b/cdx_toolkit/repackage/writers/s3.py similarity index 98% rename from cdx_toolkit/repackage/s3_writer.py rename to cdx_toolkit/repackage/writers/s3.py index 012a486..0535a64 100644 --- a/cdx_toolkit/repackage/s3_writer.py +++ b/cdx_toolkit/repackage/writers/s3.py @@ -1,9 +1,7 @@ import logging from typing import List, Dict, Optional -from cdx_toolkit.repackage.s3_utils import ( - with_retries, -) +from cdx_toolkit.repackage.retries import with_retries logger = logging.getLogger(__name__) diff --git a/cdx_toolkit/repackage/writers/series.py b/cdx_toolkit/repackage/writers/series.py new file mode 100644 index 0000000..873aeef --- /dev/null +++ b/cdx_toolkit/repackage/writers/series.py @@ -0,0 +1,41 @@ +''' +The output of a run: the series of WARC files [-]-NNN.warc.gz. +''' +import glob +import posixpath +import re +from typing import List, Optional + +import fsspec + + +def generate_warc_filename( + dest_prefix: str, + sequence: int, + writer_subprefix: Optional[str] = None, + gzip: bool = False, +) -> str: + """Generate a WARC file name from a prefix (can be a full path), an optional subprefix + (used to distinguish per-process shards), and a sequence index (file-rotation counter).""" + file_name = dest_prefix + '-' + if writer_subprefix is not None: + file_name += writer_subprefix + '-' + file_name += f'{sequence:03d}.warc' # TODO default warc command uses ".extracted.warc" + if gzip: + file_name += '.gz' + + return file_name + + +def existing_warc_files(prefix_path: str, writer_subprefix: Optional[str] = None, gzip: bool = True) -> List[str]: + """The files of the series [-]-NNN.warc[.gz] (async WARC fetcher) or + [-]-NNNNNN.extracted.warc[.gz] (--wb) that already exist (local, s3://, hf://, ...).""" + fs, fs_prefix = fsspec.url_to_fs(prefix_path) + name = re.escape(posixpath.basename(fs_prefix)) + '-' + if writer_subprefix is not None: + name += re.escape(writer_subprefix) + '-' + name += r'\d{3,}(\.extracted)?\.warc' + (r'\.gz' if gzip else '') + return sorted( + fs.unstrip_protocol(path) for path in fs.glob(glob.escape(fs_prefix) + '-*') + if re.fullmatch(name, posixpath.basename(path)) + ) diff --git a/tests/fake_warc_server.py b/tests/fake_warc_server.py index 49bce77..b4b4754 100644 --- a/tests/fake_warc_server.py +++ b/tests/fake_warc_server.py @@ -5,7 +5,7 @@ from warcio import WARCWriter -from cdx_toolkit.repackage.warc_utils import get_bytes_from_warc_record +from cdx_toolkit.repackage.warc_records import get_bytes_from_warc_record def warc_record_bytes(url: str) -> bytes: diff --git a/tests/repackage/test_data_classes.py b/tests/repackage/test_data_classes.py index 7b36a8a..00d5b06 100644 --- a/tests/repackage/test_data_classes.py +++ b/tests/repackage/test_data_classes.py @@ -1,6 +1,6 @@ import pytest -from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.http_range import RangeError, check_range_response +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.readers.http import RangeError, check_range_response def test_get_s3_bucket_and_key_from_http_job(): diff --git a/tests/repackage/test_hf_utils.py b/tests/repackage/test_hf_reader.py similarity index 97% rename from tests/repackage/test_hf_utils.py rename to tests/repackage/test_hf_reader.py index 6b92b00..7099da5 100644 --- a/tests/repackage/test_hf_utils.py +++ b/tests/repackage/test_hf_reader.py @@ -5,15 +5,15 @@ import pytest -from cdx_toolkit.repackage.hf_utils import ( +from cdx_toolkit.repackage.readers.hf import ( HfHttpReader, make_hf_reader, signed_url_expiry, - is_hf_url, hf_fs_path, hf_resolve_url, retry_after_seconds, ) +from cdx_toolkit.repackage.urls import is_hf_url def test_is_hf_url(): @@ -183,7 +183,7 @@ def test_waits_on_resolver_rate_limit(): async def no_sleep(_): return None - with patch('cdx_toolkit.repackage.hf_utils.asyncio.sleep', side_effect=no_sleep): + with patch('cdx_toolkit.repackage.readers.hf.asyncio.sleep', side_effect=no_sleep): assert asyncio.run(_read(reader, WARC_A, 0, 4)) == DATA[0:4] assert reader.throttle_count == 2 @@ -233,7 +233,7 @@ async def run(): await reader.aclose() return data - with caplog.at_level('INFO', logger='cdx_toolkit.repackage.hf_utils'): + with caplog.at_level('INFO', logger='cdx_toolkit.repackage.readers.hf'): assert asyncio.run(run()) == DATA[0:4] assert reader.throttle_count == 1 assert 'HF reader: 1 records read with 1 resolve requests' in caplog.text diff --git a/tests/repackage/test_records_source.py b/tests/repackage/test_jobs.py similarity index 93% rename from tests/repackage/test_records_source.py rename to tests/repackage/test_jobs.py index 52d90f2..02a0e5d 100644 --- a/tests/repackage/test_records_source.py +++ b/tests/repackage/test_jobs.py @@ -1,4 +1,4 @@ -from cdx_toolkit.repackage.records_source import join_warc_url +from cdx_toolkit.repackage.jobs import join_warc_url def test_join_warc_url_relative(): diff --git a/tests/repackage/test_local_writer.py b/tests/repackage/test_local_output.py similarity index 97% rename from tests/repackage/test_local_writer.py rename to tests/repackage/test_local_output.py index c8e5dce..c4461a2 100644 --- a/tests/repackage/test_local_writer.py +++ b/tests/repackage/test_local_output.py @@ -3,7 +3,7 @@ import pytest -from cdx_toolkit.repackage.local_writer import LocalFileWriter +from cdx_toolkit.repackage.writers.local import LocalFileWriter def run(coro): diff --git a/tests/repackage/test_warc_filter.py b/tests/repackage/test_pipeline.py similarity index 85% rename from tests/repackage/test_warc_filter.py rename to tests/repackage/test_pipeline.py index 4f4509b..e33b58c 100644 --- a/tests/repackage/test_warc_filter.py +++ b/tests/repackage/test_pipeline.py @@ -1,8 +1,7 @@ -from cdx_toolkit.repackage.data_classes import ThroughputTracker from tests.conftest import TEST_DATA_PATH -from cdx_toolkit.repackage.warc_filter import WARCFilter -from cdx_toolkit.repackage.records_source import RecordsSource +from cdx_toolkit.repackage.pipeline import ThroughputTracker, WARCFilter +from cdx_toolkit.repackage.jobs import RecordsSource fixture_path = TEST_DATA_PATH / 'repackage' @@ -13,7 +12,7 @@ def test_log_writer(caplog): """Test log writer.""" import logging - caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.warc_filter') + caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.pipeline') warc_filter = WARCFilter( source=_FAKE_SOURCE, @@ -32,7 +31,7 @@ def test_log_writer(caplog): def test_log_reader(caplog): """Test log reader.""" import logging - caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.warc_filter') + caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.pipeline') warc_filter = WARCFilter( source=_FAKE_SOURCE, diff --git a/tests/repackage/test_producer.py b/tests/repackage/test_producer.py index f3e9fc3..6d104d4 100644 --- a/tests/repackage/test_producer.py +++ b/tests/repackage/test_producer.py @@ -3,8 +3,8 @@ import pytest -from cdx_toolkit.repackage.data_classes import RangeJob -from cdx_toolkit.repackage.warc_filter import WARCFilter +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.pipeline import WARCFilter class FailingSource: diff --git a/tests/repackage/test_s3_utils.py b/tests/repackage/test_retries.py similarity index 97% rename from tests/repackage/test_s3_utils.py rename to tests/repackage/test_retries.py index 5e159eb..714cb1a 100644 --- a/tests/repackage/test_s3_utils.py +++ b/tests/repackage/test_retries.py @@ -3,11 +3,8 @@ import asyncio -from cdx_toolkit.repackage.s3_utils import ( - _backoff, - parse_s3_uri, - with_retries, -) +from cdx_toolkit.repackage.retries import _backoff, with_retries +from cdx_toolkit.repackage.urls import parse_s3_uri # the retry tests raise botocore's exceptions (S3 support is optional: cdx_toolkit[s3]) pytest.importorskip('botocore') diff --git a/tests/repackage/test_s3_writer.py b/tests/repackage/test_s3_output.py similarity index 98% rename from tests/repackage/test_s3_writer.py rename to tests/repackage/test_s3_output.py index 6637c39..67c5603 100644 --- a/tests/repackage/test_s3_writer.py +++ b/tests/repackage/test_s3_output.py @@ -3,7 +3,7 @@ import pytest -from cdx_toolkit.repackage.s3_writer import S3ShardWriter, mpu_abort +from cdx_toolkit.repackage.writers.s3 import S3ShardWriter, mpu_abort def test_mpu_abort_success(): diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 7777148..567d477 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -17,11 +17,11 @@ from cdx_toolkit import myrequests from cdx_toolkit.repackage import multiprocess -from cdx_toolkit.repackage.data_classes import RangeJob, RepackageResult -from cdx_toolkit.repackage.http_range import RangeError -from cdx_toolkit.repackage.local_writer import LocalFileWriter -from cdx_toolkit.repackage.records_source import RecordsSource -from cdx_toolkit.repackage.warc_filter import WARCFilter +from cdx_toolkit.repackage.jobs import RangeJob, RepackageResult +from cdx_toolkit.repackage.readers.http import RangeError +from cdx_toolkit.repackage.writers.local import LocalFileWriter +from cdx_toolkit.repackage.jobs import RecordsSource +from cdx_toolkit.repackage.pipeline import WARCFilter from tests.conftest import InlineExecutor from tests.fake_warc_server import FakeWarcServer from tests.conftest import requires_aioboto3 @@ -59,7 +59,7 @@ def flaky_get(url, headers=None, **kwargs): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=flaky_get): + with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=flaky_get): assert wf.filter() == RepackageResult(requested=4, written=3, failed=1) assert len(response_records(glob.glob(str(tmp_path / 'out-*.warc.gz')))) == 3 @@ -78,7 +78,7 @@ def test_invalid_range_responses_are_not_written(tmp_path, response): content=b'x' * length) wf = WARCFilter(source=RecordsSource([capture], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=response): + with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=response): assert wf.filter() == RepackageResult(requested=1, written=0, failed=1) assert glob.glob(str(tmp_path / 'out-*')) == [] @@ -100,7 +100,7 @@ async def write(self, data): wf = WARCFilter(source=RecordsSource(CAPTURES[:2], 'https://data.example.org', sort=False), prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=2) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=get), \ + with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=get), \ patch.object(LocalFileWriter, 'write', write): assert wf.filter() == RepackageResult(requested=2, written=2) records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) @@ -120,7 +120,7 @@ async def failing_write(self, data): raise OSError('disk full') await original_write(self, data) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get), \ + with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get), \ patch.object(LocalFileWriter, 'write', failing_write), pytest.raises(OSError, match='disk full'): wf.filter() # the first file is complete, the second was removed @@ -143,7 +143,7 @@ async def failing_flush(self): raise OSError('disk full') await original_flush(self) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get), \ + with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get), \ patch.object(LocalFileWriter, '_flush', failing_flush), pytest.raises(OSError, match='disk full'): wf.filter() assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == expected_files @@ -152,7 +152,7 @@ async def failing_flush(self): def test_rotation_by_size(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, max_file_size=1) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): assert wf.filter().written == 4 paths = sorted(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert [os.path.basename(p) for p in paths] == [f'out-00{i}.warc.gz' for i in range(1, 5)] @@ -255,7 +255,7 @@ def get(*args, **kwargs): finished.set() with patch('cdx_toolkit.myrequests.requests.get', side_effect=failing_get), \ - patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=get): + patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=get): job = RangeJob(url='https://down.example.org/a.warc.gz', offset=0, length=1) cancel_http_read_after_the_first_request(job, started) assert finished.wait(timeout=2), 'the request thread is still waiting to retry' @@ -265,7 +265,7 @@ def get(*args, **kwargs): def test_hf_reads_use_the_hf_reader(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'hf://buckets/commoncrawl/commoncrawl'), prefix_path=str(tmp_path / 'out'), writer_info={}) - with patch('cdx_toolkit.repackage.warc_filter.make_hf_reader', fake_make_hf_reader): + with patch('cdx_toolkit.repackage.readers.make_hf_reader', fake_make_hf_reader): assert wf.filter().written == 4 records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert sorted(r.rec_headers.get_header('WARC-Target-URI') for r in records) == sorted(c['url'] for c in CAPTURES) @@ -345,8 +345,8 @@ def mixed_scheme_captures(): def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): source = RecordsSource(mixed_scheme_captures(), 'https://data.example.org') with patch('aioboto3.Session') as session, \ - patch('cdx_toolkit.repackage.warc_filter.make_hf_reader', fake_make_hf_reader), \ - patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get), \ + patch('cdx_toolkit.repackage.readers.make_hf_reader', fake_make_hf_reader), \ + patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get), \ patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor): session.return_value.client.return_value = FakeS3Client() if processes == 1: @@ -369,7 +369,7 @@ def test_s3_output_with_metadata_records(tmp_path): metadata.write_bytes(b'https://example.com/\n') s3 = FakeS3Client() with patch('aioboto3.Session') as session, \ - patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): session.return_value.client.return_value = s3 result = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path='s3://bucket/out', writer_info={}, write_paths_as_metadata_records=[str(metadata)]).filter() @@ -383,7 +383,7 @@ def test_s3_output_with_metadata_records(tmp_path): def test_failing_metadata_record_removes_the_file(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, write_paths_as_metadata_records=[str(tmp_path / 'missing.txt')]) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get), \ + with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get), \ pytest.raises(FileNotFoundError): wf.filter() assert glob.glob(str(tmp_path / 'out-*')) == [] @@ -445,7 +445,7 @@ def test_s3_reads_retry_the_whole_transfer(): @requires_aioboto3 def test_no_s3_client_without_s3(tmp_path): with patch('aioboto3.Session') as session, \ - patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}).filter() session.assert_not_called() @@ -455,7 +455,7 @@ def test_no_s3_client_without_s3(tmp_path): def test_multiprocess_repackage(tmp_path, remote): prefix = 'memory://mp-test/out/RUN' if remote else str(tmp_path / 'RUN') with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ - patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): n = multiprocess.run_multiprocess_repackage( source=RecordsSource(CAPTURES, 'https://data.example.org'), n_processes=3, readers_per_process=2, prefix_path=prefix, writer_info={'isPartOf': 'test'}, writer_subprefix='SUB', @@ -486,7 +486,7 @@ def test_multiprocess_repackage_keeps_the_order_of_the_source(tmp_path): """Workers do not sort their shard again (--no-sort streams, a sorted source is sorted already).""" captures = CAPTURES[::-1] with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ - patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=fake_get): + patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): multiprocess.run_multiprocess_repackage( source=RecordsSource(captures, 'https://data.example.org', sort=False), n_processes=1, readers_per_process=1, prefix_path=str(tmp_path / 'RUN'), writer_info={}, writer_subprefix=None, diff --git a/tests/repackage/test_warc_utils.py b/tests/repackage/test_warc_records.py similarity index 91% rename from tests/repackage/test_warc_utils.py rename to tests/repackage/test_warc_records.py index cf77c4c..f04fa23 100644 --- a/tests/repackage/test_warc_utils.py +++ b/tests/repackage/test_warc_records.py @@ -1,7 +1,8 @@ import fsspec import pytest -from cdx_toolkit.repackage.warc_utils import existing_warc_files, get_metadata_record_from_path +from cdx_toolkit.repackage.warc_records import get_metadata_record_from_path +from cdx_toolkit.repackage.writers.series import existing_warc_files from tests.conftest import TEST_DATA_PATH diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py index 029d561..1a82f50 100644 --- a/tests/unit/test_aws.py +++ b/tests/unit/test_aws.py @@ -130,8 +130,8 @@ def test_s3fs_uses_the_defaults(no_user_config, monkeypatch): @requires_aioboto3 def test_warc_fetcher_s3_clients(no_user_config): - from cdx_toolkit.repackage.records_source import RecordsSource - from cdx_toolkit.repackage.warc_filter import WARCFilter + from cdx_toolkit.repackage.jobs import RecordsSource + from cdx_toolkit.repackage.pipeline import WARCFilter wf = WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=4) read_config, write_config = wf.s3_read_config(), wf.s3_write_config() diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 9451723..725684a 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -7,12 +7,12 @@ from warcio.archiveiterator import ArchiveIterator import cdx_toolkit.cli.repackage -import cdx_toolkit.repackage.warc_utils +import cdx_toolkit.repackage.warc_records import cdx_toolkit.warc from cdx_toolkit.cli import main from cdx_toolkit.cli.common import parse_size from cdx_toolkit.cli.deprecated import get_command, rewrite_deprecated_command, translate -from cdx_toolkit.repackage.data_classes import RepackageResult +from cdx_toolkit.repackage.jobs import RepackageResult import cdx_toolkit.cli.index @@ -30,7 +30,7 @@ def _make_fake_record(): ) -RECORD = cdx_toolkit.repackage.warc_utils.get_bytes_from_warc_record(_make_fake_record(), gzip=True) +RECORD = cdx_toolkit.repackage.warc_records.get_bytes_from_warc_record(_make_fake_record(), gzip=True) def _range_response(offset=10): @@ -68,7 +68,7 @@ def test_repackage_fetches_by_warc_location(tmp_path): captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) resp = _range_response() - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=resp) as get: + with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=resp) as get: main(['repackage', captures, '--prefix', str(tmp_path / 'out'), '--warc-download-prefix', 'https://x.org/']) assert get.call_args.args == ('https://x.org/crawl-data/x.warc.gz',) @@ -91,7 +91,7 @@ def test_repackage_rejects_existing_warc_files_with_and_without_wb(tmp_path, cap wb_args = ['--wb', 'ia'] if wb else [] with patch.object(cdx_toolkit.cli.repackage, 'fetch_wb_warc') as fetch, \ - patch('cdx_toolkit.repackage.data_classes.myrequests_get') as get, \ + patch('cdx_toolkit.repackage.jobs.myrequests_get') as get, \ pytest.raises(SystemExit) as exc: main(['repackage', captures, '--prefix', str(tmp_path / 'out')] + wb_args) @@ -126,7 +126,7 @@ def test_repackage_default_prefix(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=_range_response()): + with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=_range_response()): main(['repackage', captures]) assert [p.name for p in tmp_path.glob('*.warc.gz')] == ['REPACKAGE-001.warc.gz'] @@ -143,7 +143,7 @@ def test_index_to_parquet_and_repackage(tmp_path, capsys): main(['index', 'cc', '--url', 'example.com/*', '--output-format', 'parquet']) assert 'cannot be written to stdout' in capsys.readouterr().err - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=_range_response()): + with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=_range_response()): main(['repackage', str(captures), '--prefix', str(tmp_path / 'out')]) with open(tmp_path / 'out-001.warc.gz', 'rb') as f: assert [r.rec_type for r in ArchiveIterator(f)] == ['warcinfo', 'response'] @@ -152,7 +152,7 @@ def test_index_to_parquet_and_repackage(tmp_path, capsys): def test_repackage_options_are_passed_to_the_warc_fetcher(tmp_path): captures = _write_captures(tmp_path / 'captures.jsonl', []) - with patch('cdx_toolkit.repackage.warc_filter.WARCFilter') as warc_filter: + with patch('cdx_toolkit.repackage.pipeline.WARCFilter') as warc_filter: warc_filter.return_value.filter.return_value = RepackageResult() main([ 'repackage', captures, '--prefix', 'FOO', '--subprefix', 'BAR', '--size', '1MiB', @@ -290,7 +290,7 @@ def get(url, headers=None, **kwargs): raise ConnectionError('network down') return _range_response(offset=int(headers['Range'][len('bytes='):].split('-')[0])) - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=get), \ + with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=get), \ pytest.raises(SystemExit) as exc: main(['repackage', path, '--prefix', str(tmp_path / 'out')]) assert exc.value.code == 1 @@ -301,8 +301,8 @@ def get(url, headers=None, **kwargs): def test_repackage_fails_if_the_output_cannot_be_created(tmp_path): path = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) response = _range_response() - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', return_value=response), \ - patch('cdx_toolkit.repackage.local_writer.aiofiles.open', side_effect=PermissionError('read-only')), \ + with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=response), \ + patch('cdx_toolkit.repackage.writers.local.aiofiles.open', side_effect=PermissionError('read-only')), \ pytest.raises(PermissionError): main(['repackage', path, '--prefix', str(tmp_path / 'out')]) assert list(tmp_path.glob('out-*')) == [] diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index 43ce19c..6fdcd2a 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -332,7 +332,7 @@ def test_sql_without_url_column_to_repackage(parquet_index, tmp_path, capsys): assert [json.loads(line) for line in captures.read_text().splitlines()] == [ {'filename': 'a.warc.gz', 'offset': capture['offset'], 'length': capture['length']} ] - with patch('cdx_toolkit.repackage.data_classes.myrequests_get', side_effect=server.get): + with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=server.get): main(['repackage', str(captures), '--prefix', str(tmp_path / 'out'), '--warc-download-prefix', 'https://data.example.org']) assert len(list(tmp_path.glob('out-*.warc.gz'))) == 1 diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py index d8edebb..d4e5a75 100644 --- a/tests/unit/test_usage_errors.py +++ b/tests/unit/test_usage_errors.py @@ -15,7 +15,7 @@ from cdx_toolkit.indexes.backends.duckdb import DuckDbBackend from cdx_toolkit.indexes.backends.sql_builder import quote_sql_string, sql_timestamp from cdx_toolkit.record_io import RecordWriter, import_pyarrow, read_records, write_records -from cdx_toolkit.repackage.data_classes import RangeJob +from cdx_toolkit.repackage.jobs import RangeJob from tests.conftest import TEST_DATA_PATH From ac152949b4222f84eaf61c35a40720fda0e38952 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 15:10:47 +0200 Subject: [PATCH 44/91] refactor(repackage): RangeReader per storage (HTTP, S3, HF); RangeJob is plain data HttpRangeReader, S3RangeReader and HfRangeReader (was HfHttpReader) implement RangeReader.read(job); Readers picks the reader by the URL scheme of the job (for_url), so neither RangeJob nor the pipeline branches on it. --- cdx_toolkit/repackage/jobs.py | 88 +------------------- cdx_toolkit/repackage/pipeline.py | 9 +- cdx_toolkit/repackage/readers/__init__.py | 49 +++++++++-- cdx_toolkit/repackage/readers/hf.py | 18 +++- cdx_toolkit/repackage/readers/http.py | 23 +++++ cdx_toolkit/repackage/readers/s3.py | 38 +++++++++ tests/repackage/test_data_classes.py | 34 -------- tests/repackage/test_hf_reader.py | 36 ++++---- tests/repackage/test_producer.py | 5 +- tests/repackage/test_readers.py | 60 +++++++++++++ tests/repackage/test_warc_filter_pipeline.py | 67 +++++++-------- tests/unit/test_cli.py | 12 +-- tests/unit/test_sql_indexes.py | 2 +- tests/unit/test_usage_errors.py | 18 ---- 14 files changed, 242 insertions(+), 217 deletions(-) create mode 100644 cdx_toolkit/repackage/readers/s3.py delete mode 100644 tests/repackage/test_data_classes.py create mode 100644 tests/repackage/test_readers.py diff --git a/cdx_toolkit/repackage/jobs.py b/cdx_toolkit/repackage/jobs.py index 40dde84..84c2862 100644 --- a/cdx_toolkit/repackage/jobs.py +++ b/cdx_toolkit/repackage/jobs.py @@ -2,14 +2,8 @@ The captures to repackage: index records (as written by `cdxt index`) -> range jobs, and the result of a run. ''' import logging -import threading from dataclasses import dataclass -from typing import Iterable, Iterator, Mapping, Optional, Tuple - -from cdx_toolkit.myrequests import myrequests_get -from cdx_toolkit.repackage.readers.http import RangeError, check_range_response, run_in_daemon_thread -from cdx_toolkit.repackage.retries import RETRYABLE_ERRORS, with_retries -from cdx_toolkit.repackage.urls import is_hf_url, is_s3_url, parse_s3_uri +from typing import Iterable, Iterator, Mapping, Optional logger = logging.getLogger(__name__) @@ -31,7 +25,7 @@ def __add__(self, other: 'RepackageResult') -> 'RepackageResult': @dataclass(frozen=True) class RangeJob: - """Defines a S3 or HTTP range read request.""" + """The bytes offset .. offset+length-1 of a WARC file (one or more WARC records).""" url: str offset: int @@ -41,84 +35,6 @@ class RangeJob: # prefix), used to sort and shard the jobs by WARC file; `url` is authoritative for fetching. filename: Optional[str] = None - def is_s3(self): - return is_s3_url(self.url) - - def get_s3_bucket_and_key(self) -> Tuple[str, str]: - if self.is_s3(): - return parse_s3_uri(self.url) - else: - raise ValueError('Cannot get bucket and key from a HTTP job') - - def is_hf(self): - return is_hf_url(self.url) - - async def ranged_get_bytes( - self, - max_attempts: int, - base_backoff_seconds: float, - readers=None, - ) -> bytes: - """Ranged get request to S3 (aioboto3), an HF bucket, or HTTP; `readers` provides the S3 and HF readers.""" - offset = self.offset - length = self.length - - end = offset + length - 1 # inclusive - - if self.is_s3(): - # read from S3 - bucket, key = self.get_s3_bucket_and_key() - if readers is None: - raise ValueError('s3:// range read requires readers (none were provided)') - s3_client = await readers.s3() - - async def transfer() -> bytes: - """The whole transfer is retried: the request and reading the body (closed in any case).""" - resp = await s3_client.get_object(Bucket=bucket, Key=key, Range=f'bytes={offset}-{end}') - async with resp['Body'] as body: - data = await body.read() - if len(data) != length: - raise RangeError(f'{self.url}: requested {length} bytes from offset {offset}, got {len(data)}') - return data - - return await with_retries( - transfer, - op_name=f'ranged_get {bucket}/{key}[{offset}:{end}]', - max_attempts=max_attempts, - base_backoff_seconds=base_backoff_seconds, - retry_on=RETRYABLE_ERRORS + (RangeError,), - ) - - elif self.is_hf(): - # read from a Hugging Face bucket (hf://) via the configured HF reader - if readers is None: - raise ValueError('hf:// range read requires readers (none were provided)') - hf_reader = await readers.hf() - return await hf_reader.ranged_get( - self.url, - offset, - length, - max_attempts=max_attempts, - base_backoff_seconds=base_backoff_seconds, - ) - - else: - # read from HTTP - headers = {'Range': 'bytes={}-{}'.format(offset, end)} - - # in a thread, so that the event loop (other readers, the writer) is not blocked; myrequests_get keeps - # the minimum interval between requests to a host across threads. If this task is cancelled, the - # thread stops at its next wait (before a retry), and nothing waits for it. - stop = threading.Event() - try: - resp = await run_in_daemon_thread(myrequests_get, self.url, headers=headers, stop=stop) - finally: - stop.set() - check_range_response( - resp.status_code, resp.headers.get('Content-Range'), len(resp.content), offset, length, self.url - ) - return resp.content - @dataclass(frozen=True) class RangePayload: diff --git a/cdx_toolkit/repackage/pipeline.py b/cdx_toolkit/repackage/pipeline.py index a10d5f2..bc9cab0 100644 --- a/cdx_toolkit/repackage/pipeline.py +++ b/cdx_toolkit/repackage/pipeline.py @@ -197,7 +197,7 @@ async def filter_async(self) -> RepackageResult: return await self._run_filter_pipeline( range_jobs_queue=range_jobs_queue, warc_records_queue=warc_records_queue, - readers=Readers(stack, self.s3_read_config), + readers=Readers(stack, self.s3_read_config, self.max_attempts, self.base_backoff_seconds), write_s3_client=write_s3_client, ) @@ -363,11 +363,8 @@ async def read_warc_records( ) break # Exit loop, but still execute finally block assert isinstance(job, RangeJob) - data = await job.ranged_get_bytes( - self.max_attempts, - self.base_backoff_seconds, - readers=readers, - ) + reader = await readers.for_url(job.url) + data = await reader.read(job) tracker.add(bytes_count=len(data), records_count=job.records_count) counter += 1 diff --git a/cdx_toolkit/repackage/readers/__init__.py b/cdx_toolkit/repackage/readers/__init__.py index 14a56e8..dccfb87 100644 --- a/cdx_toolkit/repackage/readers/__init__.py +++ b/cdx_toolkit/repackage/readers/__init__.py @@ -1,35 +1,66 @@ ''' -The readers of WARC ranges, by URL scheme: HTTP(S) (http.py), S3 (aioboto3) and Hugging Face buckets (hf.py). +The readers of WARC ranges, one per storage: HTTP(S) (http.py), S3 (s3.py) and Hugging Face buckets (hf.py). ''' +from abc import ABC, abstractmethod import asyncio import contextlib from typing import Callable -from cdx_toolkit.repackage.readers.hf import make_hf_reader +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.urls import is_hf_url, is_s3_url + + +class RangeReader(ABC): + """Reads the bytes of a range job from its storage.""" + + @abstractmethod + async def read(self, job: RangeJob) -> bytes: + """Exactly the bytes job.offset .. job.offset + job.length - 1 of job.url; raises (after the reader's + retries) if they cannot be read, e.g. RangeError for other bytes than requested.""" + raise NotImplementedError # pragma: no cover class Readers: - """The readers of WARC ranges, by the URL scheme of a range job, each created when it is first needed: - an S3 client for s3:// and the HF reader for hf:// (http(s):// needs none). They are closed with `stack`.""" + """The reader of each range job, by the URL scheme of the job: S3 for s3://, the HF reader for hf://, + HTTP otherwise. The S3 and HF readers are created when they are first needed, and closed with `stack`.""" + + def __init__(self, stack: contextlib.AsyncExitStack, s3_config_factory: Callable, max_attempts: int = 5, + base_backoff_seconds: float = 0.5): + from cdx_toolkit.repackage.readers.http import HttpRangeReader - def __init__(self, stack: contextlib.AsyncExitStack, s3_config_factory: Callable): self.stack = stack self.s3_config_factory = s3_config_factory + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + self._http = HttpRangeReader() self._s3 = None self._hf = None self._lock = asyncio.Lock() - async def s3(self): + async def for_url(self, url: str) -> RangeReader: + if is_s3_url(url): + return await self._s3_reader() + if is_hf_url(url): + return await self._hf_reader() + return self._http + + async def _s3_reader(self) -> RangeReader: async with self._lock: if self._s3 is None: import aioboto3 + from cdx_toolkit.repackage.readers.s3 import S3RangeReader + client = aioboto3.Session().client('s3', config=self.s3_config_factory()) - self._s3 = await self.stack.enter_async_context(client) + client = await self.stack.enter_async_context(client) + self._s3 = S3RangeReader(client, self.max_attempts, self.base_backoff_seconds) return self._s3 - async def hf(self): + async def _hf_reader(self) -> RangeReader: async with self._lock: if self._hf is None: - self._hf = await self.stack.enter_async_context(make_hf_reader()) + from cdx_toolkit.repackage.readers import hf + + reader = hf.make_hf_reader(self.max_attempts, self.base_backoff_seconds) + self._hf = await self.stack.enter_async_context(reader) return self._hf diff --git a/cdx_toolkit/repackage/readers/hf.py b/cdx_toolkit/repackage/readers/hf.py index 46ab3d2..7a97c32 100644 --- a/cdx_toolkit/repackage/readers/hf.py +++ b/cdx_toolkit/repackage/readers/hf.py @@ -23,6 +23,8 @@ from typing import Tuple from urllib.parse import parse_qs, urlparse +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.readers import RangeReader from cdx_toolkit.repackage.readers.http import check_range_response from cdx_toolkit.repackage.urls import HF_SCHEME, is_hf_url @@ -82,7 +84,7 @@ def signed_url_expiry(url: str, default_ttl: float) -> float: return time.time() + default_ttl -class HfHttpReader: +class HfRangeReader(RangeReader): """Async ranged ``GET`` requests against the HF CDN (aiohttp), resolving each WARC file once. The ``/resolve/`` URL of a WARC file is requested once (``HEAD``, no redirect @@ -108,9 +110,11 @@ class HfHttpReader: # Number of signed URLs kept; the input is sorted by WARC file, so few are in use at a time. MAX_CACHED_URLS = 4096 - def __init__(self, session, token=None): + def __init__(self, session, token=None, max_attempts: int = 5, base_backoff_seconds: float = 0.5): self.session = session self.token = token + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds self.throttle_count = 0 # number of 429s seen self.throttle_wait_s = 0.0 # cumulative time spent backing off 429s self.resolve_count = 0 # number of /resolve/ requests @@ -181,6 +185,12 @@ async def _signed_url(self, resolve: str, *, refresh: bool, **kwargs) -> str: self._locks.pop(old, None) return url + async def read(self, job: RangeJob) -> bytes: + return await self.ranged_get( + job.url, job.offset, job.length, max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + ) + async def ranged_get( self, url: str, @@ -241,7 +251,7 @@ async def aclose(self): @contextlib.asynccontextmanager -async def make_hf_reader(): +async def make_hf_reader(max_attempts: int = 5, base_backoff_seconds: float = 0.5): """Async context manager yielding an HF reader that owns an ``aiohttp`` session. Imports are lazy so the optional ``hf`` extra is only required when actually @@ -253,7 +263,7 @@ async def make_hf_reader(): timeout = aiohttp.ClientTimeout(total=300, sock_connect=15) connector = aiohttp.TCPConnector(limit=0, ttl_dns_cache=300) async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session: - reader = HfHttpReader(session, token=get_token()) + reader = HfRangeReader(session, get_token(), max_attempts, base_backoff_seconds) try: yield reader finally: diff --git a/cdx_toolkit/repackage/readers/http.py b/cdx_toolkit/repackage/readers/http.py index 0a8cd8d..696faac 100644 --- a/cdx_toolkit/repackage/readers/http.py +++ b/cdx_toolkit/repackage/readers/http.py @@ -6,6 +6,10 @@ import threading from typing import Optional +from cdx_toolkit.myrequests import myrequests_get +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.readers import RangeReader + class RangeError(OSError): """A range request returned other bytes than requested.""" @@ -59,3 +63,22 @@ def run(): threading.Thread(target=run, daemon=True).start() return await future + + +class HttpRangeReader(RangeReader): + """Range requests with myrequests_get (retries, and the minimum interval between requests to a host, also + across threads).""" + + async def read(self, job: RangeJob) -> bytes: + headers = {'Range': f'bytes={job.offset}-{job.offset + job.length - 1}'} + # in a thread, so that the event loop (other readers, the writer) is not blocked. If this task is + # cancelled, the thread stops at its next wait (before a retry), and nothing waits for it. + stop = threading.Event() + try: + resp = await run_in_daemon_thread(myrequests_get, job.url, headers=headers, stop=stop) + finally: + stop.set() + check_range_response( + resp.status_code, resp.headers.get('Content-Range'), len(resp.content), job.offset, job.length, job.url + ) + return resp.content diff --git a/cdx_toolkit/repackage/readers/s3.py b/cdx_toolkit/repackage/readers/s3.py new file mode 100644 index 0000000..df2f44f --- /dev/null +++ b/cdx_toolkit/repackage/readers/s3.py @@ -0,0 +1,38 @@ +''' +Range reads from S3 (aioboto3). +''' +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.readers import RangeReader +from cdx_toolkit.repackage.readers.http import RangeError +from cdx_toolkit.repackage.retries import RETRYABLE_ERRORS, with_retries +from cdx_toolkit.repackage.urls import parse_s3_uri + + +class S3RangeReader(RangeReader): + """Ranged get_object requests with an aioboto3 S3 client.""" + + def __init__(self, client, max_attempts: int, base_backoff_seconds: float): + self.client = client + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + + async def read(self, job: RangeJob) -> bytes: + bucket, key = parse_s3_uri(job.url) + end = job.offset + job.length - 1 # inclusive + + async def transfer() -> bytes: + """The whole transfer is retried: the request and reading the body (closed in any case).""" + resp = await self.client.get_object(Bucket=bucket, Key=key, Range=f'bytes={job.offset}-{end}') + async with resp['Body'] as body: + data = await body.read() + if len(data) != job.length: + raise RangeError(f'{job.url}: requested {job.length} bytes from offset {job.offset}, got {len(data)}') + return data + + return await with_retries( + transfer, + op_name=f'ranged_get {bucket}/{key}[{job.offset}:{end}]', + max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + retry_on=RETRYABLE_ERRORS + (RangeError,), + ) diff --git a/tests/repackage/test_data_classes.py b/tests/repackage/test_data_classes.py deleted file mode 100644 index 00d5b06..0000000 --- a/tests/repackage/test_data_classes.py +++ /dev/null @@ -1,34 +0,0 @@ -import pytest -from cdx_toolkit.repackage.jobs import RangeJob -from cdx_toolkit.repackage.readers.http import RangeError, check_range_response - - -def test_get_s3_bucket_and_key_from_http_job(): - job = RangeJob( - url='http://foo.com/example', - offset=0, - length=10, - ) - with pytest.raises(ValueError): - job.get_s3_bucket_and_key() - - -@pytest.mark.parametrize('status, content_range, body_length, offset, valid', [ - (206, 'bytes 10-19/100', 10, 10, True), - (206, 'bytes 10-19/*', 10, 10, True), - (200, None, 10, 0, True), # the whole file is the requested range - (200, None, 100, 10, False), # range ignored - (206, 'bytes 11-20/100', 10, 10, False), # wrong range - (206, 'bytes 10-19/100', 9, 10, False), # truncated body - (206, None, 10, 10, False), - (302, None, 0, 10, False), # redirects are not followed -]) -def test_check_range_response(status, content_range, body_length, offset, valid): - def check(): - check_range_response(status, content_range, body_length, offset, 10, 'https://example.org/x') - - if valid: - check() - else: - with pytest.raises(RangeError): - check() diff --git a/tests/repackage/test_hf_reader.py b/tests/repackage/test_hf_reader.py index 7099da5..c0eda4a 100644 --- a/tests/repackage/test_hf_reader.py +++ b/tests/repackage/test_hf_reader.py @@ -6,7 +6,7 @@ import pytest from cdx_toolkit.repackage.readers.hf import ( - HfHttpReader, + HfRangeReader, make_hf_reader, signed_url_expiry, hf_fs_path, @@ -64,7 +64,7 @@ def test_retry_after_falls_back_to_default(): assert retry_after_seconds({'Retry-After': 'soon'}, default=3.0) == 3.0 -# --- HfHttpReader: resolve each WARC file once, read the records from the signed CDN URL --- +# --- HfRangeReader: resolve each WARC file once, read the records from the signed CDN URL --- WARC_A = 'hf://buckets/commoncrawl/commoncrawl/crawl-data/a.warc.gz' WARC_B = 'hf://buckets/commoncrawl/commoncrawl/crawl-data/b.warc.gz' @@ -128,7 +128,7 @@ def test_signed_url_expiry(): def test_resolves_each_file_once(): session = FakeSession() - reader = HfHttpReader(session, token='tok') + reader = HfRangeReader(session, token='tok') async def run(): return [await _read(reader, url, o, 10) for url, o in [(WARC_A, 0), (WARC_A, 100), (WARC_B, 5), (WARC_A, 50)]] @@ -146,7 +146,7 @@ async def run(): def test_concurrent_reads_share_one_resolve(): session = FakeSession() - reader = HfHttpReader(session) + reader = HfRangeReader(session) async def run(): return await asyncio.gather(*[_read(reader, WARC_A, o, 4) for o in range(0, 80, 8)]) @@ -156,8 +156,8 @@ async def run(): def test_resolves_again_before_expiry(): - session = FakeSession(ttl=HfHttpReader.EXPIRY_MARGIN - 1) # signed URLs that are about to expire - reader = HfHttpReader(session) + session = FakeSession(ttl=HfRangeReader.EXPIRY_MARGIN - 1) # signed URLs that are about to expire + reader = HfRangeReader(session) async def run(): await _read(reader, WARC_A, 0, 4) @@ -169,7 +169,7 @@ async def run(): def test_resolves_again_when_cdn_rejects_signed_url(): session = FakeSession(get_statuses=[403]) - reader = HfHttpReader(session) + reader = HfRangeReader(session) assert asyncio.run(_read(reader, WARC_A, 10, 4)) == DATA[10:14] assert len(session.heads) == 2 # the rejected URL was resolved again @@ -178,7 +178,7 @@ def test_resolves_again_when_cdn_rejects_signed_url(): def test_waits_on_resolver_rate_limit(): session = FakeSession(head_statuses=[429, 429]) - reader = HfHttpReader(session) + reader = HfRangeReader(session) async def no_sleep(_): return None @@ -192,7 +192,7 @@ async def no_sleep(_): def test_resolve_without_redirect_reads_from_the_resolve_url(): session = FakeSession(head_statuses=[200]) - reader = HfHttpReader(session, token='tok') + reader = HfRangeReader(session, token='tok') assert asyncio.run(_read(reader, WARC_A, 0, 4)) == DATA[0:4] url, headers = session.gets[0] @@ -202,11 +202,11 @@ def test_resolve_without_redirect_reads_from_the_resolve_url(): def test_resolve_errors_are_retried_then_raised(): # resolving is retried within each read attempt (3 x 3 attempts) session = FakeSession(head_statuses=[500] * 3) - assert asyncio.run(_read(HfHttpReader(session), WARC_A, 0, 4)) == DATA[0:4] + assert asyncio.run(_read(HfRangeReader(session), WARC_A, 0, 4)) == DATA[0:4] session = FakeSession(head_statuses=[500] * 9) with pytest.raises(OSError, match='HF HEAD .* -> 500'): - asyncio.run(_read(HfHttpReader(session), WARC_A, 0, 4)) + asyncio.run(_read(HfRangeReader(session), WARC_A, 0, 4)) assert len(session.heads) == 9 @@ -220,13 +220,13 @@ def test_resolve_errors_are_retried_then_raised(): def test_read_errors_are_retried_then_raised(get_statuses, offset, length, error): session = FakeSession(get_statuses=get_statuses) with pytest.raises(OSError, match=error): - asyncio.run(_read(HfHttpReader(session), WARC_A, offset, length)) + asyncio.run(_read(HfRangeReader(session), WARC_A, offset, length)) assert len(session.gets) == 3 def test_cdn_rate_limit_is_waited_out(caplog): session = FakeSession(get_statuses=[429]) - reader = HfHttpReader(session) + reader = HfRangeReader(session) async def run(): data = await _read(reader, WARC_A, 0, 4) @@ -241,16 +241,16 @@ async def run(): def test_gives_up_after_too_many_rate_limits(monkeypatch): - monkeypatch.setattr(HfHttpReader, 'MAX_THROTTLE_RETRIES', 0) + monkeypatch.setattr(HfRangeReader, 'MAX_THROTTLE_RETRIES', 0) session = FakeSession(get_statuses=[429, 429, 429]) with pytest.raises(OSError, match='HF 429: gave up after'): - asyncio.run(_read(HfHttpReader(session), WARC_A, 0, 4)) + asyncio.run(_read(HfRangeReader(session), WARC_A, 0, 4)) def test_signed_url_cache_is_bounded(monkeypatch): - monkeypatch.setattr(HfHttpReader, 'MAX_CACHED_URLS', 1) + monkeypatch.setattr(HfRangeReader, 'MAX_CACHED_URLS', 1) session = FakeSession() - reader = HfHttpReader(session) + reader = HfRangeReader(session) async def run(): for url in (WARC_A, WARC_B, WARC_A): @@ -273,4 +273,4 @@ async def run(): return reader reader = asyncio.run(run()) - assert isinstance(reader, HfHttpReader) and reader.token == 'tok' + assert isinstance(reader, HfRangeReader) and reader.token == 'tok' diff --git a/tests/repackage/test_producer.py b/tests/repackage/test_producer.py index 6d104d4..607b660 100644 --- a/tests/repackage/test_producer.py +++ b/tests/repackage/test_producer.py @@ -5,6 +5,7 @@ from cdx_toolkit.repackage.jobs import RangeJob from cdx_toolkit.repackage.pipeline import WARCFilter +from cdx_toolkit.repackage.readers.http import HttpRangeReader class FailingSource: @@ -15,10 +16,10 @@ def iter_range_jobs(self): def test_failing_source_fails_the_pipeline(tmp_path): """A source that raises mid-iteration stops the readers and the writer, the error propagates.""" - async def read(self, *args, **kwargs): + async def read(self, job): return b'x' wf = WARCFilter(source=FailingSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=3) - with patch.object(RangeJob, 'ranged_get_bytes', read), pytest.raises(RuntimeError, match='boom'): + with patch.object(HttpRangeReader, 'read', read), pytest.raises(RuntimeError, match='boom'): wf.filter() assert glob.glob(str(tmp_path / 'out-*')) == [] diff --git a/tests/repackage/test_readers.py b/tests/repackage/test_readers.py new file mode 100644 index 0000000..1a5e83a --- /dev/null +++ b/tests/repackage/test_readers.py @@ -0,0 +1,60 @@ +import asyncio +import contextlib +from unittest.mock import patch + +import pytest + +from cdx_toolkit.repackage.readers import Readers +from cdx_toolkit.repackage.readers.http import HttpRangeReader, RangeError, check_range_response + +from tests.conftest import requires_aioboto3 + + +@requires_aioboto3 +def test_readers_by_url_scheme(): + from cdx_toolkit.repackage.readers.s3 import S3RangeReader + + hf_reader = object() + + @contextlib.asynccontextmanager + async def fake_make_hf_reader(max_attempts, base_backoff_seconds): + assert (max_attempts, base_backoff_seconds) == (3, 0.1) + yield hf_reader + + async def main(): + async with contextlib.AsyncExitStack() as stack: + readers = Readers(stack, lambda: None, max_attempts=3, base_backoff_seconds=0.1) + http = await readers.for_url('https://data.commoncrawl.org/a.warc.gz') + s3 = await readers.for_url('s3://commoncrawl/a.warc.gz') + hf = await readers.for_url('hf://buckets/commoncrawl/commoncrawl/a.warc.gz') + # one reader per scheme + assert await readers.for_url('s3://commoncrawl/b.warc.gz') is s3 + assert await readers.for_url('http://example.org/b.warc.gz') is http + return http, s3, hf + + with patch('aioboto3.Session'), patch('cdx_toolkit.repackage.readers.hf.make_hf_reader', fake_make_hf_reader): + http, s3, hf = asyncio.run(main()) + assert isinstance(http, HttpRangeReader) + assert isinstance(s3, S3RangeReader) and (s3.max_attempts, s3.base_backoff_seconds) == (3, 0.1) + assert hf is hf_reader + + +@pytest.mark.parametrize('status, content_range, body_length, offset, valid', [ + (206, 'bytes 10-19/100', 10, 10, True), + (206, 'bytes 10-19/*', 10, 10, True), + (200, None, 10, 0, True), # the whole file is the requested range + (200, None, 100, 10, False), # range ignored + (206, 'bytes 11-20/100', 10, 10, False), # wrong range + (206, 'bytes 10-19/100', 9, 10, False), # truncated body + (206, None, 10, 10, False), + (302, None, 0, 10, False), # redirects are not followed +]) +def test_check_range_response(status, content_range, body_length, offset, valid): + def check(): + check_range_response(status, content_range, body_length, offset, 10, 'https://example.org/x') + + if valid: + check() + else: + with pytest.raises(RangeError): + check() diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 567d477..eab552b 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -18,7 +18,8 @@ from cdx_toolkit import myrequests from cdx_toolkit.repackage import multiprocess from cdx_toolkit.repackage.jobs import RangeJob, RepackageResult -from cdx_toolkit.repackage.readers.http import RangeError +from cdx_toolkit.repackage.readers.http import HttpRangeReader, RangeError +from cdx_toolkit.repackage.readers.s3 import S3RangeReader from cdx_toolkit.repackage.writers.local import LocalFileWriter from cdx_toolkit.repackage.jobs import RecordsSource from cdx_toolkit.repackage.pipeline import WARCFilter @@ -59,7 +60,7 @@ def flaky_get(url, headers=None, **kwargs): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}) - with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=flaky_get): + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=flaky_get): assert wf.filter() == RepackageResult(requested=4, written=3, failed=1) assert len(response_records(glob.glob(str(tmp_path / 'out-*.warc.gz')))) == 3 @@ -78,7 +79,7 @@ def test_invalid_range_responses_are_not_written(tmp_path, response): content=b'x' * length) wf = WARCFilter(source=RecordsSource([capture], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}) - with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=response): + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=response): assert wf.filter() == RepackageResult(requested=1, written=0, failed=1) assert glob.glob(str(tmp_path / 'out-*')) == [] @@ -100,7 +101,7 @@ async def write(self, data): wf = WARCFilter(source=RecordsSource(CAPTURES[:2], 'https://data.example.org', sort=False), prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=2) - with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=get), \ + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=get), \ patch.object(LocalFileWriter, 'write', write): assert wf.filter() == RepackageResult(requested=2, written=2) records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) @@ -120,7 +121,7 @@ async def failing_write(self, data): raise OSError('disk full') await original_write(self, data) - with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get), \ + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ patch.object(LocalFileWriter, 'write', failing_write), pytest.raises(OSError, match='disk full'): wf.filter() # the first file is complete, the second was removed @@ -143,7 +144,7 @@ async def failing_flush(self): raise OSError('disk full') await original_flush(self) - with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get), \ + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ patch.object(LocalFileWriter, '_flush', failing_flush), pytest.raises(OSError, match='disk full'): wf.filter() assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == expected_files @@ -152,7 +153,7 @@ async def failing_flush(self): def test_rotation_by_size(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, max_file_size=1) - with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): assert wf.filter().written == 4 paths = sorted(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert [os.path.basename(p) for p in paths] == [f'out-00{i}.warc.gz' for i in range(1, 5)] @@ -173,8 +174,8 @@ def iter_range_jobs(self): reads = [] - async def blocking_read(self, *args, **kwargs): - reads.append(self.url) + async def blocking_read(self, job): + reads.append(job.url) if len(reads) > 1: await asyncio.Event().wait() # blocks forever return b'x' @@ -192,7 +193,7 @@ async def run(): await asyncio.wait_for(task, timeout=2) def main(): - with patch.object(RangeJob, 'ranged_get_bytes', blocking_read): + with patch.object(HttpRangeReader, 'read', blocking_read): asyncio.run(run()) thread = threading.Thread(target=main, daemon=True) @@ -206,7 +207,7 @@ def main(): def cancel_http_read_after_the_first_request(job, request_started): """Cancel the read once the first request has started; returns how long asyncio.run() took after cancelling.""" async def main(): - task = asyncio.create_task(job.ranged_get_bytes(1, 0)) + task = asyncio.create_task(HttpRangeReader().read(job)) while not request_started.is_set(): await asyncio.sleep(0.01) task.cancel() @@ -255,7 +256,7 @@ def get(*args, **kwargs): finished.set() with patch('cdx_toolkit.myrequests.requests.get', side_effect=failing_get), \ - patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=get): + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=get): job = RangeJob(url='https://down.example.org/a.warc.gz', offset=0, length=1) cancel_http_read_after_the_first_request(job, started) assert finished.wait(timeout=2), 'the request thread is still waiting to retry' @@ -265,7 +266,7 @@ def get(*args, **kwargs): def test_hf_reads_use_the_hf_reader(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'hf://buckets/commoncrawl/commoncrawl'), prefix_path=str(tmp_path / 'out'), writer_info={}) - with patch('cdx_toolkit.repackage.readers.make_hf_reader', fake_make_hf_reader): + with patch('cdx_toolkit.repackage.readers.hf.make_hf_reader', fake_make_hf_reader): assert wf.filter().written == 4 records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert sorted(r.rec_headers.get_header('WARC-Target-URI') for r in records) == sorted(c['url'] for c in CAPTURES) @@ -324,12 +325,12 @@ async def read(self): class FakeHfReader: - async def ranged_get(self, url, offset, length, **kwargs): - return SERVER.files[url.split('/', 5)[-1]][offset:offset + length] + async def read(self, job): + return SERVER.files[job.url.split('/', 5)[-1]][job.offset:job.offset + job.length] @contextlib.asynccontextmanager -async def fake_make_hf_reader(): +async def fake_make_hf_reader(*args): yield FakeHfReader() @@ -345,8 +346,8 @@ def mixed_scheme_captures(): def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): source = RecordsSource(mixed_scheme_captures(), 'https://data.example.org') with patch('aioboto3.Session') as session, \ - patch('cdx_toolkit.repackage.readers.make_hf_reader', fake_make_hf_reader), \ - patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get), \ + patch('cdx_toolkit.repackage.readers.hf.make_hf_reader', fake_make_hf_reader), \ + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor): session.return_value.client.return_value = FakeS3Client() if processes == 1: @@ -369,7 +370,7 @@ def test_s3_output_with_metadata_records(tmp_path): metadata.write_bytes(b'https://example.com/\n') s3 = FakeS3Client() with patch('aioboto3.Session') as session, \ - patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): session.return_value.client.return_value = s3 result = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path='s3://bucket/out', writer_info={}, write_paths_as_metadata_records=[str(metadata)]).filter() @@ -383,21 +384,21 @@ def test_s3_output_with_metadata_records(tmp_path): def test_failing_metadata_record_removes_the_file(tmp_path): wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, write_paths_as_metadata_records=[str(tmp_path / 'missing.txt')]) - with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get), \ + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ pytest.raises(FileNotFoundError): wf.filter() assert glob.glob(str(tmp_path / 'out-*')) == [] -class FakeS3Readers: - """An S3 client whose responses have the given bodies (bytes, or an exception raised when reading).""" +class FakeS3Responses: + """An S3 client (client()) whose responses have the given bodies (bytes, or an exception raised when reading).""" def __init__(self, *bodies): self.bodies = list(bodies) self.requests = 0 self.closed = 0 - async def s3(self): + def client(self): readers = self class Body: @@ -426,26 +427,26 @@ async def get_object(self, **kwargs): def test_s3_reads_check_the_length(): job = RangeJob(url='s3://commoncrawl/crawl-data/a.warc.gz', offset=0, length=10) with pytest.raises(RangeError, match='requested 10 bytes'): - asyncio.run(job.ranged_get_bytes(1, 0, readers=FakeS3Readers(b'short'))) - with pytest.raises(ValueError, match='requires readers'): - asyncio.run(job.ranged_get_bytes(1, 0)) + asyncio.run(S3RangeReader(FakeS3Responses(b'short').client(), 1, 0).read(job)) @requires_aioboto3 def test_s3_reads_retry_the_whole_transfer(): from botocore.exceptions import ReadTimeoutError - readers = FakeS3Readers(ReadTimeoutError(endpoint_url='s3://x'), ConnectionResetError(), b'short', b'0123456789') + responses = FakeS3Responses( + ReadTimeoutError(endpoint_url='s3://x'), ConnectionResetError(), b'short', b'0123456789' + ) job = RangeJob(url='s3://commoncrawl/crawl-data/a.warc.gz', offset=0, length=10) - assert asyncio.run(job.ranged_get_bytes(5, 0, readers=readers)) == b'0123456789' - assert readers.requests == 4 - assert readers.closed == 4 # also the failed responses + assert asyncio.run(S3RangeReader(responses.client(), 5, 0).read(job)) == b'0123456789' + assert responses.requests == 4 + assert responses.closed == 4 # also the failed responses @requires_aioboto3 def test_no_s3_client_without_s3(tmp_path): with patch('aioboto3.Session') as session, \ - patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}).filter() session.assert_not_called() @@ -455,7 +456,7 @@ def test_no_s3_client_without_s3(tmp_path): def test_multiprocess_repackage(tmp_path, remote): prefix = 'memory://mp-test/out/RUN' if remote else str(tmp_path / 'RUN') with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ - patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): n = multiprocess.run_multiprocess_repackage( source=RecordsSource(CAPTURES, 'https://data.example.org'), n_processes=3, readers_per_process=2, prefix_path=prefix, writer_info={'isPartOf': 'test'}, writer_subprefix='SUB', @@ -486,7 +487,7 @@ def test_multiprocess_repackage_keeps_the_order_of_the_source(tmp_path): """Workers do not sort their shard again (--no-sort streams, a sorted source is sorted already).""" captures = CAPTURES[::-1] with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ - patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=fake_get): + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): multiprocess.run_multiprocess_repackage( source=RecordsSource(captures, 'https://data.example.org', sort=False), n_processes=1, readers_per_process=1, prefix_path=str(tmp_path / 'RUN'), writer_info={}, writer_subprefix=None, diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 725684a..ba1dffa 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -68,7 +68,7 @@ def test_repackage_fetches_by_warc_location(tmp_path): captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) resp = _range_response() - with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=resp) as get: + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=resp) as get: main(['repackage', captures, '--prefix', str(tmp_path / 'out'), '--warc-download-prefix', 'https://x.org/']) assert get.call_args.args == ('https://x.org/crawl-data/x.warc.gz',) @@ -91,7 +91,7 @@ def test_repackage_rejects_existing_warc_files_with_and_without_wb(tmp_path, cap wb_args = ['--wb', 'ia'] if wb else [] with patch.object(cdx_toolkit.cli.repackage, 'fetch_wb_warc') as fetch, \ - patch('cdx_toolkit.repackage.jobs.myrequests_get') as get, \ + patch('cdx_toolkit.repackage.readers.http.myrequests_get') as get, \ pytest.raises(SystemExit) as exc: main(['repackage', captures, '--prefix', str(tmp_path / 'out')] + wb_args) @@ -126,7 +126,7 @@ def test_repackage_default_prefix(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) - with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=_range_response()): + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=_range_response()): main(['repackage', captures]) assert [p.name for p in tmp_path.glob('*.warc.gz')] == ['REPACKAGE-001.warc.gz'] @@ -143,7 +143,7 @@ def test_index_to_parquet_and_repackage(tmp_path, capsys): main(['index', 'cc', '--url', 'example.com/*', '--output-format', 'parquet']) assert 'cannot be written to stdout' in capsys.readouterr().err - with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=_range_response()): + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=_range_response()): main(['repackage', str(captures), '--prefix', str(tmp_path / 'out')]) with open(tmp_path / 'out-001.warc.gz', 'rb') as f: assert [r.rec_type for r in ArchiveIterator(f)] == ['warcinfo', 'response'] @@ -290,7 +290,7 @@ def get(url, headers=None, **kwargs): raise ConnectionError('network down') return _range_response(offset=int(headers['Range'][len('bytes='):].split('-')[0])) - with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=get), \ + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=get), \ pytest.raises(SystemExit) as exc: main(['repackage', path, '--prefix', str(tmp_path / 'out')]) assert exc.value.code == 1 @@ -301,7 +301,7 @@ def get(url, headers=None, **kwargs): def test_repackage_fails_if_the_output_cannot_be_created(tmp_path): path = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) response = _range_response() - with patch('cdx_toolkit.repackage.jobs.myrequests_get', return_value=response), \ + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=response), \ patch('cdx_toolkit.repackage.writers.local.aiofiles.open', side_effect=PermissionError('read-only')), \ pytest.raises(PermissionError): main(['repackage', path, '--prefix', str(tmp_path / 'out')]) diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index 6fdcd2a..1587875 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -332,7 +332,7 @@ def test_sql_without_url_column_to_repackage(parquet_index, tmp_path, capsys): assert [json.loads(line) for line in captures.read_text().splitlines()] == [ {'filename': 'a.warc.gz', 'offset': capture['offset'], 'length': capture['length']} ] - with patch('cdx_toolkit.repackage.jobs.myrequests_get', side_effect=server.get): + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=server.get): main(['repackage', str(captures), '--prefix', str(tmp_path / 'out'), '--warc-download-prefix', 'https://data.example.org']) assert len(list(tmp_path.glob('out-*.warc.gz'))) == 1 diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py index d4e5a75..63a7bcf 100644 --- a/tests/unit/test_usage_errors.py +++ b/tests/unit/test_usage_errors.py @@ -1,5 +1,4 @@ """Usage errors and small edge cases of `cdxt index` / `cdxt repackage` and their helpers.""" -import asyncio import io import runpy import sys @@ -15,7 +14,6 @@ from cdx_toolkit.indexes.backends.duckdb import DuckDbBackend from cdx_toolkit.indexes.backends.sql_builder import quote_sql_string, sql_timestamp from cdx_toolkit.record_io import RecordWriter, import_pyarrow, read_records, write_records -from cdx_toolkit.repackage.jobs import RangeJob from tests.conftest import TEST_DATA_PATH @@ -120,19 +118,3 @@ def test_parquet_batches_exactly_full(tmp_path, monkeypatch): path = str(tmp_path / 'r.parquet') assert write_records(iter([{'url': str(i)} for i in range(4)]), path, 'parquet') == 4 assert [r['url'] for r in read_records(path)] == ['0', '1', '2', '3'] - - -def test_hf_range_job_needs_a_reader(): - job = RangeJob(url='hf://buckets/ns/name/x.warc.gz', offset=0, length=1) - with pytest.raises(ValueError, match='requires readers'): - asyncio.run(job.ranged_get_bytes(1, 0)) - - class Reader: - async def ranged_get(self, url, offset, length, **kwargs): - return b'x' - - class Readers: - async def hf(self): - return Reader() - - assert asyncio.run(job.ranged_get_bytes(1, 0, readers=Readers())) == b'x' From 0b66613d4e18aa2232a503acdf734e0749f1b30d Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 15:15:18 +0200 Subject: [PATCH 45/91] refactor(repackage): OutputFile base class; WarcFileSeries owns numbering, rotation and the warcinfo record LocalOutputFile and S3OutputFile (were LocalFileWriter and S3ShardWriter) implement OutputFile, whose life cycle is documented once; open_output_file picks one by path. WarcFileSeries writes the series of WARC files (first file with the first record, rotation at --size, warcinfo and metadata records, abort on failure), so the writer task of the pipeline only passes the records on. --- cdx_toolkit/repackage/pipeline.py | 123 ++++--------------- cdx_toolkit/repackage/writers/__init__.py | 71 ++++++----- cdx_toolkit/repackage/writers/local.py | 6 +- cdx_toolkit/repackage/writers/s3.py | 39 +++--- cdx_toolkit/repackage/writers/series.py | 118 +++++++++++++++++- tests/repackage/test_local_output.py | 14 +-- tests/repackage/test_s3_output.py | 5 +- tests/repackage/test_series.py | 99 +++++++++++++++ tests/repackage/test_warc_filter_pipeline.py | 14 +-- 9 files changed, 318 insertions(+), 171 deletions(-) create mode 100644 tests/repackage/test_series.py diff --git a/cdx_toolkit/repackage/pipeline.py b/cdx_toolkit/repackage/pipeline.py index bc9cab0..649982e 100644 --- a/cdx_toolkit/repackage/pipeline.py +++ b/cdx_toolkit/repackage/pipeline.py @@ -14,10 +14,7 @@ from cdx_toolkit.repackage.jobs import RangeJob, RangePayload, RepackageResult from cdx_toolkit.repackage.readers import Readers from cdx_toolkit.repackage.urls import is_s3_url -from cdx_toolkit.repackage.warc_records import ( - get_bytes_from_warc_record, get_metadata_record_from_path, warcinfo_record_bytes, -) -from cdx_toolkit.repackage.writers import new_warc_writer +from cdx_toolkit.repackage.writers.series import WarcFileSeries _STOP = object() @@ -140,9 +137,7 @@ def __init__( self.warc_version = warc_version self.min_part_size = min_part_size self.max_file_size = max_file_size - # Number of the next output file, given the current one (0 before the first file). The - # multi-process mode shares one counter across processes, so file names are final right away. - self.next_file_sequence = next_file_sequence or (lambda sequence: sequence + 1) + self.next_file_sequence = next_file_sequence # see WarcFileSeries def filter(self) -> RepackageResult: """Perform the filtering process (calls async method via asyncio.run). @@ -379,38 +374,12 @@ async def read_warc_records( return {'reader_id': reader_id, 'stats': tracker.get_stats(), 'failed': failed} - async def write_metadata_records(self, writer, warcinfo_id: str) -> int: - """Write WARC metadata records based on paths""" - records_size = 0 - records_count = 0 - - logger.info(f'Writing {len(self.write_paths_as_metadata_records)} metadata records to WARC ... ') - - # Metadata records are written at the beginning each WARC file. - for i, resource_record_path in enumerate(self.write_paths_as_metadata_records): - logger.info(f'Writing metadata record from {resource_record_path} ...') - record = get_metadata_record_from_path( - file_path=resource_record_path, - warcinfo_id=warcinfo_id, - ) - record_data = get_bytes_from_warc_record(record, warc_version=self.warc_version, gzip=self.gzip) - await writer.write(record_data) - - # Keep track but do not rotate resource records - records_size += len(record_data) - records_count += 1 - - logger.info(f'Metadata records added: {records_count}') - - return records_size - async def write_warc_records( self, warc_records_queue: asyncio.Queue, s3_client=None, ) -> dict: - """Write WARC records. The single writer owns the output WARC (one shard per - process) and appends ranges to it, rotating by --size when set. + """Write the WARC records to the series of output files (one series per process), rotating at --size. Args: warc_records_queue: Queue to read RangePayload objects from. @@ -419,19 +388,26 @@ async def write_warc_records( Returns: dict: Statistics dictionary with throughput stats. """ - # File rotation tracking; the first file is only created with the first record, so - # no WARC file is written without records - writer = None - current_file_sequence = 0 - current_file_size = 0 - records_in_file = 0 - + series = WarcFileSeries( + self.prefix_path, + self.writer_info, + writer_subprefix=self.writer_subprefix, + metadata_paths=self.write_paths_as_metadata_records, + warc_version=self.warc_version, + gzip=self.gzip, + max_file_size=self.max_file_size, + next_file_sequence=self.next_file_sequence, + s3_client=s3_client, + min_part_size=self.min_part_size, + max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + ) tracker = ThroughputTracker() tracker.start() counter = 0 - # Response records; any failure is fatal, also when closing a file: the current file is aborted - # (removed, or its S3 upload aborted), the files completed before stay + # any failure is fatal, also when closing a file: the current file is aborted (removed, or its S3 upload + # aborted), the files completed before stay try: while True: item = await warc_records_queue.get() @@ -447,40 +423,13 @@ async def write_warc_records( break assert isinstance(item, RangePayload) counter += 1 - - if writer is None: - # first file, with header and metadata records - current_file_sequence = self.next_file_sequence(0) - writer, current_file_size = await self.start_file(current_file_sequence, s3_client) - records_in_file = 0 - elif records_in_file and self.max_file_size and \ - current_file_size + len(item.data) > self.max_file_size: - # rotate due to the size limit (a file gets at least one record, also if the record is - # larger than the size limit); the finished file is not aborted if starting the next fails - await writer.close() - writer = None - current_file_sequence = self.next_file_sequence(current_file_sequence) - writer, current_file_size = await self.start_file(current_file_sequence, s3_client) - records_in_file = 0 - logger.info(f'Rotated to new WARC file sequence {current_file_sequence} due to size limit') - - # Write actual response record - await writer.write(item.data) - records_in_file += 1 - current_file_size += len(item.data) + await series.write(item.data) tracker.add(bytes_count=len(item.data), records_count=item.job.records_count) - - # Log progress every N items self.log_writer(counter=counter, tracker=tracker) - - if writer is not None: - await writer.close() + await series.close() except BaseException: - # also if closing failed: a writer that cannot close removes its file (or aborts its upload) itself, - # and abort() is a no-op then logger.error('WARC writer failed, aborting the current WARC file') - if writer is not None: - await writer.abort() + await series.abort() raise return {'stats': tracker.get_stats()} @@ -508,31 +457,3 @@ def log_writer(self, counter: int, tracker: ThroughputTracker): stats['total_bytes'] / (1024 * 1024), stats['mb_per_sec'], ) - - async def start_file(self, sequence: int, s3_client=None): - """Start a new WARC file with its warcinfo record (and the metadata records, if any); if that fails, the - file is aborted. - - Returns the writer and the number of bytes written.""" - writer, warc_file_name = new_warc_writer( - sequence, - self.prefix_path, - max_attempts=self.max_attempts, - base_backoff_seconds=self.base_backoff_seconds, - min_part_size=self.min_part_size, - writer_subprefix=self.writer_subprefix, - gzip=self.gzip, - s3_client=s3_client, - ) - logger.debug('Starting WARC file %s', warc_file_name) - try: - await writer.start() - header, warcinfo_id = warcinfo_record_bytes(warc_file_name, self.writer_info, self.warc_version, self.gzip) - await writer.write(header) - file_size = len(header) - if self.write_paths_as_metadata_records: - file_size += await self.write_metadata_records(writer, warcinfo_id=warcinfo_id) - except BaseException: - await writer.abort() - raise - return writer, file_size diff --git a/cdx_toolkit/repackage/writers/__init__.py b/cdx_toolkit/repackage/writers/__init__.py index f23ef98..6b8b235 100644 --- a/cdx_toolkit/repackage/writers/__init__.py +++ b/cdx_toolkit/repackage/writers/__init__.py @@ -1,34 +1,45 @@ ''' -The writers of output files: a local file (local.py) or an S3 object written with a multipart upload (s3.py). +The output files of cdxt repackage: a local file (local.py) or an S3 object written with a multipart upload +(s3.py), and the series of WARC files they make up (series.py). ''' -import os -from typing import Optional, Tuple, Union +from abc import ABC, abstractmethod from cdx_toolkit.repackage.urls import is_s3_url, parse_s3_uri -from cdx_toolkit.repackage.writers.local import LocalFileWriter -from cdx_toolkit.repackage.writers.s3 import S3ShardWriter -from cdx_toolkit.repackage.writers.series import generate_warc_filename - - -def new_warc_writer( - sequence: int, - output_path_prefix: str, - max_attempts: int, - base_backoff_seconds: float, - min_part_size: int, - writer_subprefix: Optional[str] = None, - gzip: bool = False, - s3_client=None, -) -> Tuple[Union[S3ShardWriter, LocalFileWriter], str]: - """A new (not started) writer of a WARC file, local or S3, and the file's name (for the warcinfo record). - - Both writers have the same life cycle: start(), write()..., then close(), which completes the file, or - abort(), which removes it (or aborts its upload); a writer that cannot close aborts itself.""" - if is_s3_url(output_path_prefix): - dest_bucket, dest_prefix = parse_s3_uri(output_path_prefix) - key = generate_warc_filename(dest_prefix, sequence=sequence, writer_subprefix=writer_subprefix, gzip=gzip) - writer = S3ShardWriter(s3_client, key, dest_bucket, min_part_size, max_attempts, base_backoff_seconds) - return writer, key.split('/')[-1] - - path = generate_warc_filename(output_path_prefix, sequence=sequence, writer_subprefix=writer_subprefix, gzip=gzip) - return LocalFileWriter(file_path=path), os.path.basename(path) # with backslashes on Windows + + +class OutputFile(ABC): + """One output file, written in one go: start(), write()..., then close(). + + close() completes the file; if that fails, the file aborts itself (and close() raises). abort() removes the + incomplete file (or aborts its upload), also after a failed start(); it can be called more than once.""" + + @abstractmethod + async def start(self) -> None: + raise NotImplementedError # pragma: no cover + + @abstractmethod + async def write(self, data: bytes) -> None: + raise NotImplementedError # pragma: no cover + + @abstractmethod + async def close(self) -> None: + raise NotImplementedError # pragma: no cover + + @abstractmethod + async def abort(self) -> None: + raise NotImplementedError # pragma: no cover + + +def open_output_file(path: str, s3_client=None, min_part_size: int = 5 * 1024 * 1024, max_attempts: int = 5, + base_backoff_seconds: float = 0.5) -> OutputFile: + """A new (not started) output file: an S3 object for s3://bucket/key (written with `s3_client`), else a local + file.""" + if is_s3_url(path): + from cdx_toolkit.repackage.writers.s3 import S3OutputFile + + bucket, key = parse_s3_uri(path) + return S3OutputFile(s3_client, bucket, key, min_part_size, max_attempts, base_backoff_seconds) + + from cdx_toolkit.repackage.writers.local import LocalOutputFile + + return LocalOutputFile(path) diff --git a/cdx_toolkit/repackage/writers/local.py b/cdx_toolkit/repackage/writers/local.py index e33df09..46ff19c 100644 --- a/cdx_toolkit/repackage/writers/local.py +++ b/cdx_toolkit/repackage/writers/local.py @@ -3,11 +3,11 @@ import aiofiles +from cdx_toolkit.repackage.writers import OutputFile -class LocalFileWriter: - """Async writer for local file system using aiofiles. - Like S3ShardWriter: close() completes the file, or removes it and raises; abort() removes it.""" +class LocalOutputFile(OutputFile): + """A local file, written with aiofiles; abort() removes it.""" def __init__(self, file_path: str, buffer_size: int = 8192, mode: str = 'wb'): self.file_path = file_path diff --git a/cdx_toolkit/repackage/writers/s3.py b/cdx_toolkit/repackage/writers/s3.py index 0535a64..583a094 100644 --- a/cdx_toolkit/repackage/writers/s3.py +++ b/cdx_toolkit/repackage/writers/s3.py @@ -2,6 +2,7 @@ from typing import List, Dict, Optional from cdx_toolkit.repackage.retries import with_retries +from cdx_toolkit.repackage.writers import OutputFile logger = logging.getLogger(__name__) @@ -79,8 +80,8 @@ async def mpu_abort(s3, bucket: str, key: str, upload_id: str): logger.exception('Failed to abort MPU %s on %s/%s', upload_id, bucket, key) -class S3ShardWriter: - """Manages one MPU: buffers bytes, uploads >=5 MiB parts, completes on close. +class S3OutputFile(OutputFile): + """An S3 object written with a multipart upload (MPU): buffers bytes, uploads >=5 MiB parts, completes on close. A part that cannot be uploaded (after the retries) makes the writer fail: the MPU is aborted, so no object with missing bytes is created, and close() raises instead of completing the upload.""" @@ -88,15 +89,15 @@ class S3ShardWriter: def __init__( self, s3_client, - shard_key: str, - dest_bucket: str, + bucket: str, + key: str, min_part_size: int, max_attempts: int, base_backoff_seconds: float, ): self.s3_client = s3_client - self.shard_key = shard_key - self.dest_bucket = dest_bucket + self.bucket = bucket + self.key = key self.min_part_size = min_part_size self.max_attempts = max_attempts self.base_backoff_seconds = base_backoff_seconds @@ -109,22 +110,22 @@ def __init__( async def start(self): self.upload_id = await mpu_create( self.s3_client, - self.dest_bucket, - self.shard_key, + self.bucket, + self.key, max_attempts=self.max_attempts, base_backoff_seconds=self.base_backoff_seconds, ) - logger.info('Started MPU for %s (UploadId=%s)', self.shard_key, self.upload_id) + logger.info('Started MPU for %s (UploadId=%s)', self.key, self.upload_id) async def _upload_part(self, size: int): """Upload the first `size` bytes of the buffer as the next part; they leave the buffer only once uploaded.""" if self.failed: - raise RuntimeError(f'the upload of {self.shard_key} failed before') + raise RuntimeError(f'the upload of {self.key} failed before') try: etag = await mpu_upload_part( self.s3_client, - self.dest_bucket, - self.shard_key, + self.bucket, + self.key, self.upload_id, self.part_number, bytes(self.buffer[:size]), @@ -146,23 +147,23 @@ async def write(self, data: bytes): async def close(self): """Upload the rest and complete the MPU; if that fails, the MPU is aborted.""" if self.failed: - raise RuntimeError(f'the upload of {self.shard_key} failed, it was aborted') + raise RuntimeError(f'the upload of {self.key} failed, it was aborted') if self.buffer: await self._upload_part(len(self.buffer)) # aborts on failure try: if self.parts: await mpu_complete( self.s3_client, - self.dest_bucket, - self.shard_key, + self.bucket, + self.key, self.upload_id, self.parts, self.max_attempts, self.base_backoff_seconds, ) - logger.info('Completed MPU for %s with %d parts.', self.shard_key, len(self.parts)) + logger.info('Completed MPU for %s with %d parts.', self.key, len(self.parts)) except BaseException: - logger.exception('Completing MPU failed for %s; attempting abort.', self.shard_key) + logger.exception('Completing MPU failed for %s; attempting abort.', self.key) await self.abort() raise @@ -173,5 +174,5 @@ async def abort(self): self.failed = True self.buffer.clear() if self.upload_id: - logger.warning('Aborting MPU for %s', self.shard_key) - await mpu_abort(self.s3_client, self.dest_bucket, self.shard_key, self.upload_id) + logger.warning('Aborting MPU for %s', self.key) + await mpu_abort(self.s3_client, self.bucket, self.key, self.upload_id) diff --git a/cdx_toolkit/repackage/writers/series.py b/cdx_toolkit/repackage/writers/series.py index 873aeef..62fedbf 100644 --- a/cdx_toolkit/repackage/writers/series.py +++ b/cdx_toolkit/repackage/writers/series.py @@ -2,12 +2,23 @@ The output of a run: the series of WARC files [-]-NNN.warc.gz. ''' import glob +import logging +import os import posixpath import re -from typing import List, Optional +from typing import Callable, Dict, List, Optional import fsspec +from cdx_toolkit.repackage.urls import is_s3_url +from cdx_toolkit.repackage.warc_records import ( + get_bytes_from_warc_record, get_metadata_record_from_path, warcinfo_record_bytes, +) +from cdx_toolkit.repackage.writers import OutputFile, open_output_file + + +logger = logging.getLogger(__name__) + def generate_warc_filename( dest_prefix: str, @@ -39,3 +50,108 @@ def existing_warc_files(prefix_path: str, writer_subprefix: Optional[str] = None fs.unstrip_protocol(path) for path in fs.glob(glob.escape(fs_prefix) + '-*') if re.fullmatch(name, posixpath.basename(path)) ) + + +class WarcFileSeries: + """The WARC files [-]-NNN.warc.gz of a run, written one after the other. + + The first file is started with the first record, so no file is written without records. Each file starts with + its warcinfo record (and the metadata records, if any); a new file is started when the next record would make + the current one larger than `max_file_size` (each file gets at least one record). + + A failure is fatal: the file it happened in is aborted (removed, or its upload aborted), the files completed + before are kept.""" + + def __init__( + self, + prefix_path: str, + writer_info: Dict, + writer_subprefix: Optional[str] = None, + metadata_paths: Optional[List[str]] = None, + warc_version: str = '1.0', + gzip: bool = True, + max_file_size: Optional[int] = None, + next_file_sequence: Optional[Callable[[int], int]] = None, + s3_client=None, + min_part_size: int = 5 * 1024 * 1024, + max_attempts: int = 5, + base_backoff_seconds: float = 0.5, + ): + self.prefix_path = prefix_path + self.writer_info = writer_info + self.writer_subprefix = writer_subprefix + self.metadata_paths = metadata_paths or [] + self.warc_version = warc_version + self.gzip = gzip + self.max_file_size = max_file_size + # Number of the next file, given the current one (0 before the first file). The multi-process mode + # shares one counter across processes, so file names are final right away. + self.next_file_sequence = next_file_sequence or (lambda sequence: sequence + 1) + self.s3_client = s3_client + self.min_part_size = min_part_size + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + + self.sequence = 0 # of the current (or last) file + self._file: Optional[OutputFile] = None + self._file_size = 0 + self._records_in_file = 0 + + async def write(self, record: bytes) -> None: + """Write a (complete, possibly gzipped) WARC record, in a new file if needed.""" + if self._file is None: + await self._start_file(self.next_file_sequence(0)) + elif self._records_in_file and self.max_file_size and self._file_size + len(record) > self.max_file_size: + await self._close_file() # the finished file is kept if starting the next one fails + await self._start_file(self.next_file_sequence(self.sequence)) + logger.info(f'Rotated to new WARC file sequence {self.sequence} due to size limit') + await self._file.write(record) + self._records_in_file += 1 + self._file_size += len(record) + + async def close(self) -> None: + """Complete the current file (if it cannot be completed, it is aborted).""" + if self._file is not None: + await self._close_file() + + async def abort(self) -> None: + """Abort the current file (no-op if there is none, e.g. after a failure that aborted it already).""" + file, self._file = self._file, None + if file is not None: + await file.abort() + + async def _close_file(self) -> None: + file, self._file = self._file, None + await file.close() # aborts itself if it fails + + async def _start_file(self, sequence: int) -> None: + """Start the file with its warcinfo record (and the metadata records); if that fails, it is aborted.""" + path = generate_warc_filename(self.prefix_path, sequence, self.writer_subprefix, self.gzip) + file = open_output_file(path, self.s3_client, self.min_part_size, self.max_attempts, self.base_backoff_seconds) + # the name in the warcinfo record (local paths may have backslashes on Windows) + name = posixpath.basename(path) if is_s3_url(path) else os.path.basename(path) + logger.debug('Starting WARC file %s', name) + try: + await file.start() + header, warcinfo_id = warcinfo_record_bytes(name, self.writer_info, self.warc_version, self.gzip) + await file.write(header) + size = len(header) + if self.metadata_paths: + size += await self._write_metadata_records(file, warcinfo_id) + except BaseException: + await file.abort() + raise + self._file, self.sequence, self._file_size, self._records_in_file = file, sequence, size, 0 + + async def _write_metadata_records(self, file: OutputFile, warcinfo_id: str) -> int: + """Write a metadata record for each of the metadata paths; returns their size.""" + size = 0 + logger.info(f'Writing {len(self.metadata_paths)} metadata records to WARC ... ') + for path in self.metadata_paths: + logger.info(f'Writing metadata record from {path} ...') + record = get_metadata_record_from_path(file_path=path, warcinfo_id=warcinfo_id) + data = get_bytes_from_warc_record(record, warc_version=self.warc_version, gzip=self.gzip) + await file.write(data) + size += len(data) + logger.info(f'Metadata records added: {len(self.metadata_paths)}') + return size diff --git a/tests/repackage/test_local_output.py b/tests/repackage/test_local_output.py index c4461a2..dcc17ba 100644 --- a/tests/repackage/test_local_output.py +++ b/tests/repackage/test_local_output.py @@ -3,7 +3,7 @@ import pytest -from cdx_toolkit.repackage.writers.local import LocalFileWriter +from cdx_toolkit.repackage.writers.local import LocalOutputFile def run(coro): @@ -15,7 +15,7 @@ def test_writes_buffered_and_byte_exact(tmp_path): chunks = [bytes(range(256)), b'small', b'x' * 20000, b'', b'\x00\xff' * 10] async def write(): - writer = LocalFileWriter(str(path), buffer_size=1024) + writer = LocalOutputFile(str(path), buffer_size=1024) await writer.start() await writer.write(chunks[1]) assert path.read_bytes() == b'' # buffered @@ -30,7 +30,7 @@ async def write(): def test_close_without_start(tmp_path): - run(LocalFileWriter(str(tmp_path / 'out')).close()) + run(LocalOutputFile(str(tmp_path / 'out')).close()) assert not (tmp_path / 'out').exists() @@ -38,7 +38,7 @@ def test_abort_removes_the_file(tmp_path): path = tmp_path / 'out.warc.gz' async def write_and_abort(): - writer = LocalFileWriter(str(path), buffer_size=4) + writer = LocalOutputFile(str(path), buffer_size=4) await writer.start() await writer.write(b'incomplete') await writer.abort() @@ -46,17 +46,17 @@ async def write_and_abort(): run(write_and_abort()) assert not path.exists() - run(LocalFileWriter(str(path)).abort()) # without start + run(LocalOutputFile(str(path)).abort()) # without start def test_start_fails_without_the_directory(tmp_path): with pytest.raises(FileNotFoundError): - run(LocalFileWriter(os.path.join(str(tmp_path), 'missing', 'out')).start()) + run(LocalOutputFile(os.path.join(str(tmp_path), 'missing', 'out')).start()) def test_close_removes_the_file_if_the_last_write_fails(tmp_path): async def close_with_failing_flush(): - writer = LocalFileWriter(str(tmp_path / 'out')) + writer = LocalOutputFile(str(tmp_path / 'out')) await writer.start() await writer.write(b'x') handle = writer.file_handle diff --git a/tests/repackage/test_s3_output.py b/tests/repackage/test_s3_output.py index 67c5603..1ce570b 100644 --- a/tests/repackage/test_s3_output.py +++ b/tests/repackage/test_s3_output.py @@ -3,7 +3,7 @@ import pytest -from cdx_toolkit.repackage.writers.s3 import S3ShardWriter, mpu_abort +from cdx_toolkit.repackage.writers.s3 import S3OutputFile, mpu_abort def test_mpu_abort_success(): @@ -73,8 +73,7 @@ async def abort_multipart_upload(self, **kwargs): def make_writer(s3): - writer = S3ShardWriter(s3, 'out.warc.gz', 'bucket', min_part_size=4, max_attempts=2, - base_backoff_seconds=0) + writer = S3OutputFile(s3, 'bucket', 'out.warc.gz', min_part_size=4, max_attempts=2, base_backoff_seconds=0) writer.upload_id = 'upload-id' return writer diff --git a/tests/repackage/test_series.py b/tests/repackage/test_series.py new file mode 100644 index 0000000..ae4bb97 --- /dev/null +++ b/tests/repackage/test_series.py @@ -0,0 +1,99 @@ +"""The series of output WARC files: numbering, rotation, warcinfo records, and aborting on failure.""" +import asyncio +import glob +import os +from io import BytesIO +from unittest.mock import patch + +import pytest +from warcio.archiveiterator import ArchiveIterator +from warcio.recordbuilder import RecordBuilder + +from cdx_toolkit.repackage.warc_records import get_bytes_from_warc_record +from cdx_toolkit.repackage.writers.local import LocalOutputFile +from cdx_toolkit.repackage.writers.series import WarcFileSeries + + +def record(url: str) -> bytes: + rec = RecordBuilder().create_warc_record(url, 'resource', payload=BytesIO(b'x' * 100)) + return get_bytes_from_warc_record(rec, gzip=True) + + +def records_by_file(prefix): + result = {} + for path in sorted(glob.glob(prefix + '-*.warc.gz')): + with open(path, 'rb') as f: + result[os.path.basename(path)] = [ + (r.rec_type, r.rec_headers.get_header('WARC-Filename') or r.rec_headers.get_header('WARC-Target-URI')) + for r in ArchiveIterator(f) + ] + return result + + +def write_all(series, records): + async def main(): + for data in records: + await series.write(data) + await series.close() + + asyncio.run(main()) + + +def test_no_file_without_records(tmp_path): + write_all(WarcFileSeries(str(tmp_path / 'out'), {}), []) + assert os.listdir(tmp_path) == [] + + +def test_rotation_and_warcinfo_per_file(tmp_path): + prefix = str(tmp_path / 'out') + data = [record(f'https://example.com/{i}') for i in range(3)] + # room for the warcinfo record and one record, not two + series = WarcFileSeries(prefix, {'software': 'test'}, writer_subprefix='SUB', max_file_size=len(data[0]) * 2) + write_all(series, data) + + assert records_by_file(prefix) == { + f'out-SUB-00{i + 1}.warc.gz': [('warcinfo', f'out-SUB-00{i + 1}.warc.gz'), + ('resource', f'https://example.com/{i}')] + for i in range(3) + } + assert series.sequence == 3 + + +def test_record_larger_than_the_size_limit_gets_a_file(tmp_path): + prefix = str(tmp_path / 'out') + write_all(WarcFileSeries(prefix, {}, max_file_size=1), [record('https://example.com/a')]) + assert list(records_by_file(prefix)) == ['out-001.warc.gz'] + + +def test_shared_file_numbers(tmp_path): + numbers = iter([5, 9]) + prefix = str(tmp_path / 'out') + series = WarcFileSeries(prefix, {}, max_file_size=1, next_file_sequence=lambda current: next(numbers)) + write_all(series, [record('https://example.com/a'), record('https://example.com/b')]) + assert list(records_by_file(prefix)) == ['out-005.warc.gz', 'out-009.warc.gz'] + + +def test_failure_aborts_the_current_file_only(tmp_path): + prefix = str(tmp_path / 'out') + series = WarcFileSeries(prefix, {}, max_file_size=1) + original_write = LocalOutputFile.write + + writes = [] + + async def write(self, data): + writes.append(self.file_path) + if writes.count(self.file_path) == 2 and self.file_path.endswith('-002.warc.gz'): # after the warcinfo + raise OSError('disk full') + await original_write(self, data) + + async def main(): + await series.write(record('https://example.com/a')) + try: + await series.write(record('https://example.com/b')) + except OSError: + await series.abort() + raise + + with patch.object(LocalOutputFile, 'write', write), pytest.raises(OSError, match='disk full'): + asyncio.run(main()) + assert list(records_by_file(prefix)) == ['out-001.warc.gz'] diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index eab552b..5329018 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -20,7 +20,7 @@ from cdx_toolkit.repackage.jobs import RangeJob, RepackageResult from cdx_toolkit.repackage.readers.http import HttpRangeReader, RangeError from cdx_toolkit.repackage.readers.s3 import S3RangeReader -from cdx_toolkit.repackage.writers.local import LocalFileWriter +from cdx_toolkit.repackage.writers.local import LocalOutputFile from cdx_toolkit.repackage.jobs import RecordsSource from cdx_toolkit.repackage.pipeline import WARCFilter from tests.conftest import InlineExecutor @@ -87,7 +87,7 @@ def test_invalid_range_responses_are_not_written(tmp_path, response): def test_a_slow_http_read_does_not_block_the_other_readers_and_the_writer(tmp_path): """The first read is held until the second capture has been written.""" second_written = threading.Event() - original_write = LocalFileWriter.write + original_write = LocalOutputFile.write def get(url, headers=None, **kwargs): if url.endswith(CAPTURES[0]['filename']): @@ -102,7 +102,7 @@ async def write(self, data): wf = WARCFilter(source=RecordsSource(CAPTURES[:2], 'https://data.example.org', sort=False), prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=2) with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=get), \ - patch.object(LocalFileWriter, 'write', write): + patch.object(LocalOutputFile, 'write', write): assert wf.filter() == RepackageResult(requested=2, written=2) records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert [r.rec_headers.get_header('WARC-Target-URI') for r in records] == [CAPTURES[1]['url'], CAPTURES[0]['url']] @@ -113,7 +113,7 @@ def test_writer_error_stops_the_pipeline_and_removes_the_file(tmp_path): writer_info={}, max_file_size=1) writes = [] - original_write = LocalFileWriter.write + original_write = LocalOutputFile.write async def failing_write(self, data): writes.append(data) @@ -122,7 +122,7 @@ async def failing_write(self, data): await original_write(self, data) with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ - patch.object(LocalFileWriter, 'write', failing_write), pytest.raises(OSError, match='disk full'): + patch.object(LocalOutputFile, 'write', failing_write), pytest.raises(OSError, match='disk full'): wf.filter() # the first file is complete, the second was removed assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == ['out-001.warc.gz'] @@ -136,7 +136,7 @@ def test_close_error_stops_the_pipeline_and_removes_the_file(tmp_path, failing_c wf = WARCFilter(source=RecordsSource(CAPTURES[:2], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, max_file_size=1) flushes = [] - original_flush = LocalFileWriter._flush + original_flush = LocalOutputFile._flush async def failing_flush(self): flushes.append(self.file_path) @@ -145,7 +145,7 @@ async def failing_flush(self): await original_flush(self) with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ - patch.object(LocalFileWriter, '_flush', failing_flush), pytest.raises(OSError, match='disk full'): + patch.object(LocalOutputFile, '_flush', failing_flush), pytest.raises(OSError, match='disk full'): wf.filter() assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == expected_files From ebb263112f9d3cb91480a99798802784ef299969 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 15:17:22 +0200 Subject: [PATCH 46/91] refactor(repackage): WARCFilter is only the pipeline; ThroughputTracker.summary() for the logs The reader and writer tasks return their ThroughputTracker (and the number of failed reads) instead of stats dicts; one log_progress() and ThroughputTracker.summary() replace the separate reader/writer log blocks. --- cdx_toolkit/repackage/pipeline.py | 148 +++++++++--------------------- tests/repackage/test_pipeline.py | 32 ++----- 2 files changed, 52 insertions(+), 128 deletions(-) diff --git a/cdx_toolkit/repackage/pipeline.py b/cdx_toolkit/repackage/pipeline.py index 649982e..e7820cc 100644 --- a/cdx_toolkit/repackage/pipeline.py +++ b/cdx_toolkit/repackage/pipeline.py @@ -3,11 +3,10 @@ import contextlib import logging import os -import statistics import threading import time -from dataclasses import dataclass -from typing import Callable, List, Optional, Dict +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional, Tuple from cdx_toolkit.aws import botocore_config @@ -24,16 +23,13 @@ @dataclass class ThroughputTracker: - """Track throughput metrics for fetchers and consumers.""" + """Items (range reads, or written records), bytes and WARC records of a reader or the writer, for the logs.""" - start_time: float = 0.0 + start_time: float = field(default_factory=time.time) total_bytes: int = 0 total_requests: int = 0 total_records: int = 0 - def start(self): - self.start_time = time.time() - def add(self, bytes_count: int = 0, records_count: int = 0, requests_count: int = 1): self.total_bytes += bytes_count self.total_requests += requests_count @@ -53,22 +49,24 @@ def get_stats(self) -> dict: 'records_per_sec': self.total_records / elapsed if elapsed > 0 else 0, } + def summary(self) -> str: + stats = self.get_stats() + return ( + f"{stats['total_requests']} items, {stats['total_bytes'] / (1024 * 1024):.1f} MB " + f"in {stats['elapsed']:.1f}s, {stats['mb_per_sec']:.2f} MB/s, {stats['records_per_sec']:.2f} rec/s" + ) -class WARCFilter: - """Filter or extract specific records from WARC files based on CDX indexes. - - The WARC filter uses a three stage listner-producer-consumer pattern. - Filter targets: - - CDX index files from local or remote file system, containing paths to WARC files and positions of target records. +class WARCFilter: + """Copy WARC records, byte for byte, from the WARC files of the range jobs into a series of WARC files. - WARC reader: - - HTTP range reads - - S3 range reads + A pipeline of asyncio tasks: + - the producer drives the (sync) source in a thread and puts its range jobs into a queue, + - N reader tasks read the bytes of the jobs, with the RangeReader for the URL scheme of each job + (HTTP, S3, Hugging Face buckets; see readers/), + - one writer task writes them to the WarcFileSeries (local files or S3 objects; see writers/). - WARC writer: - - Local file system - - S3 using multi-part uploads + Multi-core scaling is done with processes (see multiprocess.py), each with its own pipeline. """ def __init__( @@ -288,7 +286,7 @@ async def feed(): for task in done: task.result() # raises the exception of a failed task requested, readers_results = await feeder - writer_result = await warc_writer + writer_tracker = await warc_writer except BaseException: stop.set() for task in tasks: @@ -296,28 +294,19 @@ async def feed(): await asyncio.gather(*tasks, return_exceptions=True) raise - readers_records = sum([result['stats']['total_records'] for result in readers_results]) - readers_mb_per_sec = self.num_readers * statistics.mean( - [result['stats']['mb_per_sec'] for result in readers_results] - ) - readers_records_per_sec = self.num_readers * statistics.mean( - [result['stats']['records_per_sec'] for result in readers_results] - ) - - logger.info(f'All WARC readers completed: {readers_records} records') - logger.info(f'Total reader throughput: {readers_mb_per_sec:.2f} MB/s; {readers_records_per_sec:.2f} rec/s') - - writer_stats = writer_result['stats'] - logger.info(f"WARC writer completed: {writer_stats['total_records']} records") + reader_stats = [tracker.get_stats() for tracker, _ in readers_results] logger.info( - f"Total writer throughput: {writer_stats['mb_per_sec']:.2f} MB/s; " - f"{writer_stats['records_per_sec']:.2f} rec/s" + 'All WARC readers completed: %d records, %.2f MB/s, %.2f rec/s', + sum(stats['total_records'] for stats in reader_stats), + sum(stats['mb_per_sec'] for stats in reader_stats), + sum(stats['records_per_sec'] for stats in reader_stats), ) + logger.info('WARC writer completed: %s', writer_tracker.summary()) - failed = sum(result['failed'] for result in readers_results) + failed = sum(failed for _, failed in readers_results) if failed: logger.error('%d of %d captures could not be fetched', failed, requested) - return RepackageResult(requested=requested, written=writer_stats['total_records'], failed=failed) + return RepackageResult(requested=requested, written=writer_tracker.total_records, failed=failed) async def read_warc_records( self, @@ -325,7 +314,7 @@ async def read_warc_records( range_jobs_queue: asyncio.Queue, warc_records_queue: asyncio.Queue, readers: Readers, - ) -> dict: + ) -> Tuple[ThroughputTracker, int]: """Read WARC records based on range jobs -> enqueue RangePayload. Args: @@ -335,50 +324,34 @@ async def read_warc_records( readers: The readers of WARC ranges (by URL scheme). Returns: - dict: Statistics dictionary with reader_id and throughput stats. + The reader's throughput and the number of range jobs that could not be read. """ tracker = ThroughputTracker() - tracker.start() - counter = 0 failed = 0 while True: job = await range_jobs_queue.get() + if job is _STOP: + logger.info('WARC Reader %d stopping: %s', reader_id, tracker.summary()) + return tracker, failed + assert isinstance(job, RangeJob) try: - if job is _STOP: - stats = tracker.get_stats() - logger.info( - 'WARC Reader %d stopping. Stats: %.1fs, %d requests, %.1f MB, %.2f MB/s, %.2f req/s', - reader_id, - stats['elapsed'], - stats['total_requests'], - stats['total_bytes'] / (1024 * 1024), - stats['mb_per_sec'], - stats['requests_per_sec'], - ) - break # Exit loop, but still execute finally block - assert isinstance(job, RangeJob) reader = await readers.for_url(job.url) data = await reader.read(job) - tracker.add(bytes_count=len(data), records_count=job.records_count) - counter += 1 - - # Log progress every N items - self.log_reader(reader_id=reader_id, counter=counter, tracker=tracker) - - await warc_records_queue.put(RangePayload(job=job, data=data)) except Exception as e: # the capture is lost, the others are still fetched; the run fails at the end failed += 1 logger.error('WARC Reader %d failed on %s [%d,%d]: %r', reader_id, job.url, job.offset, job.length, e) - - return {'reader_id': reader_id, 'stats': tracker.get_stats(), 'failed': failed} + continue + tracker.add(bytes_count=len(data), records_count=job.records_count) + self.log_progress(f'WARC Reader {reader_id}', tracker) + await warc_records_queue.put(RangePayload(job=job, data=data)) async def write_warc_records( self, warc_records_queue: asyncio.Queue, s3_client=None, - ) -> dict: + ) -> ThroughputTracker: """Write the WARC records to the series of output files (one series per process), rotating at --size. Args: @@ -386,7 +359,7 @@ async def write_warc_records( s3_client: Optional S3 client for writing WARC files to S3. Returns: - dict: Statistics dictionary with throughput stats. + The writer's throughput. """ series = WarcFileSeries( self.prefix_path, @@ -403,8 +376,6 @@ async def write_warc_records( base_backoff_seconds=self.base_backoff_seconds, ) tracker = ThroughputTracker() - tracker.start() - counter = 0 # any failure is fatal, also when closing a file: the current file is aborted (removed, or its S3 upload # aborted), the files completed before stay @@ -412,48 +383,19 @@ async def write_warc_records( while True: item = await warc_records_queue.get() if item is _STOP: - stats = tracker.get_stats() - logger.info( - 'WARC writer stopping. Stats: %.1fs, %d items, %.1f MB written, %.2f MB/s write speed', - stats['elapsed'], - stats['total_requests'], - stats['total_bytes'] / (1024 * 1024), - stats['mb_per_sec'], - ) break assert isinstance(item, RangePayload) - counter += 1 await series.write(item.data) tracker.add(bytes_count=len(item.data), records_count=item.job.records_count) - self.log_writer(counter=counter, tracker=tracker) + self.log_progress('WARC Writer', tracker) await series.close() except BaseException: logger.error('WARC writer failed, aborting the current WARC file') await series.abort() raise + return tracker - return {'stats': tracker.get_stats()} - - def log_reader(self, reader_id: int, counter: int, tracker: ThroughputTracker): - """Log progress every N items.""" - if self.log_every_n > 0 and counter % self.log_every_n == 0: - stats = tracker.get_stats() - logger.info( - 'WARC Reader %d: %d items, %.1f MB, %.2f MB/s, %.2f req/s', - reader_id, - counter, - stats['total_bytes'] / (1024 * 1024), - stats['mb_per_sec'], - stats['requests_per_sec'], - ) - - def log_writer(self, counter: int, tracker: ThroughputTracker): - """Log progress every N items.""" - if self.log_every_n > 0 and counter % self.log_every_n == 0: - stats = tracker.get_stats() - logger.info( - 'WARC Writer: %d items, %.1f MB written, %.2f MB/s', - counter, - stats['total_bytes'] / (1024 * 1024), - stats['mb_per_sec'], - ) + def log_progress(self, name: str, tracker: ThroughputTracker) -> None: + """Log the progress of a reader or the writer every --log-every-n items.""" + if self.log_every_n > 0 and tracker.total_requests % self.log_every_n == 0: + logger.info('%s: %s', name, tracker.summary()) diff --git a/tests/repackage/test_pipeline.py b/tests/repackage/test_pipeline.py index e33b58c..e7f01c4 100644 --- a/tests/repackage/test_pipeline.py +++ b/tests/repackage/test_pipeline.py @@ -9,8 +9,8 @@ _FAKE_SOURCE = RecordsSource([], 'https://data.commoncrawl.org') -def test_log_writer(caplog): - """Test log writer.""" +def test_log_progress(caplog): + """Progress is logged every --log-every-n items.""" import logging caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.pipeline') @@ -21,27 +21,9 @@ def test_log_writer(caplog): log_every_n=2, ) tracker = ThroughputTracker() - warc_filter.log_writer(0, tracker) - warc_filter.log_writer(1, tracker) - warc_filter.log_writer(2, tracker) + for _ in range(5): + tracker.add(bytes_count=1024 * 1024, records_count=1) + warc_filter.log_progress('WARC Reader 1', tracker) - assert caplog.text.count('WARC Writer') == 2 - - -def test_log_reader(caplog): - """Test log reader.""" - import logging - caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.pipeline') - - warc_filter = WARCFilter( - source=_FAKE_SOURCE, - prefix_path='/fake/prefix', - writer_info={'writer_id': 1}, - log_every_n=2, - ) - tracker = ThroughputTracker() - warc_filter.log_reader(1, 0, tracker) - warc_filter.log_reader(1, 1, tracker) - warc_filter.log_reader(1, 2, tracker) - - assert caplog.text.count('WARC Reader 1') == 2 + assert caplog.text.count('WARC Reader 1: ') == 2 + assert 'WARC Reader 1: 4 items, 4.0 MB in ' in caplog.text From 57b6dc377c3b07af68c951fda7e63f656c4220af Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 15:19:19 +0200 Subject: [PATCH 47/91] refactor(myrequests): HostRateLimiter for the minimum interval between requests to a host One class (with its lock) replaces the retry_info dict and the functions around it (get_retries, update_next_fetch, reserve_fetch_time, finish_fetch); myrequests_get's behaviour is unchanged. --- cdx_toolkit/myrequests.py | 108 ++++++++----------- tests/repackage/test_warc_filter_pipeline.py | 4 +- tests/unit/test_myrequests.py | 17 ++- 3 files changed, 58 insertions(+), 71 deletions(-) diff --git a/cdx_toolkit/myrequests.py b/cdx_toolkit/myrequests.py index d3fa0bb..c097b2a 100644 --- a/cdx_toolkit/myrequests.py +++ b/cdx_toolkit/myrequests.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Dict, Optional import requests import logging import threading @@ -30,52 +30,40 @@ def dns_fatal(hostname): return True -retry_info = { - 'default': { - 'next_fetch': 0, - 'minimum_interval': DEFAULT_MIN_RETRY_INTERVAL, - }, - 'index.commoncrawl.org': { - 'next_fetch': 0, - 'minimum_interval': CC_INDEX_MIN_RETRY_INTERVAL, - }, - 'data.commoncrawl.org': { - 'next_fetch': 0, - 'minimum_interval': CC_DATA_MIN_RETRY_INTERVAL, - }, - 'web.archive.org': { - 'next_fetch': 0, - 'minimum_interval': IA_MIN_RETRY_INTERVAL, - }, -} - - -def get_retries(hostname): - if hostname not in retry_info: - retry_info[hostname] = retry_info['default'].copy() - LOGGER.debug('initializing retry info for new host ' + hostname) - entry = retry_info[hostname] - if not entry['next_fetch']: - entry['next_fetch'] = time.time() - return entry['next_fetch'], entry['minimum_interval'] +class HostRateLimiter: + """The minimum interval between the starts of requests to the same host, also when requests are made from + several threads (e.g. the parallel WARC readers of cdxt repackage).""" + def __init__(self, default_interval: float, intervals: Dict[str, float]): + self.default_interval = default_interval + self.intervals = dict(intervals) # seconds, by hostname + self._next_fetch: Dict[str, float] = {} # the earliest start of the next request, by hostname + self._lock = threading.Lock() -def update_next_fetch(hostname, next_fetch): - retry_info[hostname]['next_fetch'] = next_fetch + def minimum_interval(self, hostname: str) -> float: + return self.intervals.get(hostname, self.default_interval) + def reserve(self, hostname: str, now: float) -> float: + """The time at which the next request to `hostname` may start, at least the minimum interval after the + previous one; the time is reserved, so concurrent requests are spaced out too.""" + with self._lock: + fetch_time = max(now, self._next_fetch.get(hostname, now)) + self._next_fetch[hostname] = fetch_time + self.minimum_interval(hostname) + return fetch_time -# requests may be made from several threads (e.g. the parallel WARC readers of cdxt repackage) -_retry_lock = threading.Lock() + def finish(self, hostname: str) -> None: + """After a request (which may have taken long, with retries): the next one may start the minimum interval + from now, or later if that time is reserved already.""" + with self._lock: + next_fetch = time.time() + self.minimum_interval(hostname) + self._next_fetch[hostname] = max(self._next_fetch.get(hostname, 0), next_fetch) -def reserve_fetch_time(hostname: str, now: float) -> float: - """The time at which the next request to `hostname` may start, at least the host's minimum interval - after the previous one; the time is reserved, so concurrent requests are spaced out too.""" - with _retry_lock: - next_fetch, minimum_interval = get_retries(hostname) - fetch_time = max(now, next_fetch) - update_next_fetch(hostname, fetch_time + minimum_interval) - return fetch_time +RATE_LIMITER = HostRateLimiter(DEFAULT_MIN_RETRY_INTERVAL, { + 'index.commoncrawl.org': CC_INDEX_MIN_RETRY_INTERVAL, + 'data.commoncrawl.org': CC_DATA_MIN_RETRY_INTERVAL, + 'web.archive.org': IA_MIN_RETRY_INTERVAL, +}) class RequestCancelled(Exception): @@ -90,14 +78,6 @@ def _sleep(seconds: float, stop: Optional[threading.Event]) -> None: raise RequestCancelled() -def finish_fetch(hostname: str) -> None: - """After a request: the next one may start the minimum interval from now (or later, if reserved).""" - with _retry_lock: - get_retries(hostname) - entry = retry_info[hostname] - entry['next_fetch'] = max(entry['next_fetch'], time.time() + entry['minimum_interval']) - - def myrequests_get( url, params=None, @@ -113,8 +93,6 @@ def myrequests_get( With `stop`, the waits (before the request, between retries) end with RequestCancelled once it is set; a request in progress ends with its timeout.""" - t = time.time() - if raise_error_after_n_errors is None: raise_error_after_n_errors = MAX_ERRORS @@ -122,14 +100,8 @@ def myrequests_get( raise_warning_after_n_errors = WARNING_AFTER_N_ERRORS hostname = urlparse(url).hostname - fetch_time = reserve_fetch_time(hostname, t) # next_fetch is also updated at the bottom - minimum_interval = retry_info[hostname]['minimum_interval'] - - if t < fetch_time: - dt = fetch_time - t - if dt > 3.1: - LOGGER.debug('sleeping for {:.3f}s before next fetch'.format(dt)) - _sleep(dt, stop) + _wait_for_turn(hostname, stop) + minimum_interval = RATE_LIMITER.minimum_interval(hostname) if params: if 'from_ts' in params: @@ -212,11 +184,17 @@ def myrequests_get( LOGGER.warning('something unexpected happened, giving up after %s', str(e)) raise - hostname = urlparse(url).hostname - if hostname not in previously_seen_hostnames: - previously_seen_hostnames.add(hostname) - - # in case we had a lot of retries, etc - finish_fetch(hostname) + previously_seen_hostnames.add(hostname) + RATE_LIMITER.finish(hostname) # in case we had a lot of retries, etc return resp + + +def _wait_for_turn(hostname: str, stop: Optional[threading.Event]) -> None: + """Wait until a request to `hostname` may start (see HostRateLimiter).""" + now = time.time() + dt = RATE_LIMITER.reserve(hostname, now) - now + if dt > 0: + if dt > 3.1: + LOGGER.debug('sleeping for {:.3f}s before next fetch'.format(dt)) + _sleep(dt, stop) diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_warc_filter_pipeline.py index 5329018..2f845e5 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_warc_filter_pipeline.py @@ -221,7 +221,7 @@ async def main(): def test_cancelled_http_read_does_not_wait_for_the_request(monkeypatch): """asyncio.run() returns while the (uninterruptible) request is still blocked.""" - monkeypatch.setitem(myrequests.retry_info, 'blocked.example.org', {'next_fetch': 0, 'minimum_interval': 0}) + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'blocked.example.org', 0) started, release = threading.Event(), threading.Event() def blocking_get(*args, **kwargs): @@ -240,7 +240,7 @@ def blocking_get(*args, **kwargs): def test_cancelled_http_read_stops_retrying(monkeypatch): """The thread does not keep retrying (here: waiting 60s after a connection error) after the cancellation.""" - monkeypatch.setitem(myrequests.retry_info, 'down.example.org', {'next_fetch': 0, 'minimum_interval': 0}) + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'down.example.org', 0) started, finished = threading.Event(), threading.Event() requests_made = [] diff --git a/tests/unit/test_myrequests.py b/tests/unit/test_myrequests.py index 20a5e8e..3b3e5b1 100644 --- a/tests/unit/test_myrequests.py +++ b/tests/unit/test_myrequests.py @@ -33,15 +33,24 @@ def test_get_with_connection_error(caplog): def test_concurrent_requests_keep_the_minimum_interval(): import threading - from cdx_toolkit import myrequests + from cdx_toolkit.myrequests import HostRateLimiter host = 'interval-test.example.org' - myrequests.retry_info[host] = {'next_fetch': 1.0, 'minimum_interval': 0.5} + limiter = HostRateLimiter(3.0, {host: 0.5}) times = [] - threads = [threading.Thread(target=lambda: times.append(myrequests.reserve_fetch_time(host, 1000.0))) - for _ in range(4)] + threads = [threading.Thread(target=lambda: times.append(limiter.reserve(host, 1000.0))) for _ in range(4)] for thread in threads: thread.start() for thread in threads: thread.join() assert sorted(times) == [1000.0, 1000.5, 1001.0, 1001.5] + assert limiter.minimum_interval('other.example.org') == 3.0 + + +def test_finish_spaces_the_next_request_from_the_end_of_the_previous_one(): + from cdx_toolkit.myrequests import HostRateLimiter + + limiter = HostRateLimiter(10.0, {}) + with patch('cdx_toolkit.myrequests.time.time', return_value=2000.0): + limiter.finish('example.org') + assert limiter.reserve('example.org', 2001.0) == 2010.0 From 0825739b7c031a8623f4084e7df16cbb656a81e2 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 15:20:11 +0200 Subject: [PATCH 48/91] fix(myrequests): a DNS error is only fatal for hosts not seen before dns_fatal was given the URL instead of the hostname, so it was always true: every DNS error failed at once, also for hosts that worked before (where it may be temporary). --- CHANGELOG.md | 1 + cdx_toolkit/myrequests.py | 10 +++++----- tests/unit/test_myrequests.py | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f4aa03..3ff5eb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ if WARC files with the same prefix exist already, instead of overwriting them (also with `--wb`, which numbered its files after the existing ones before) + fixed `--to` with a day (e.g. `--to 20240410`), which was padded to the end of the month, and February in leap years + + fixed DNS errors of hosts that worked before: they are retried again (like connection errors) instead of failing at once + removed the deprecated `cdx_iter` and `cdx_size` scripts + S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual addressing, no payload signing, 2 concurrent requests) unless configured by the user; `CDXT_AWS_DEFAULTS=0` turns them off; diff --git a/cdx_toolkit/myrequests.py b/cdx_toolkit/myrequests.py index c097b2a..37725e2 100644 --- a/cdx_toolkit/myrequests.py +++ b/cdx_toolkit/myrequests.py @@ -24,10 +24,10 @@ } -def dns_fatal(hostname): - """We have a dns error, should we fail immediately or not?""" - if hostname not in previously_seen_hostnames: - return True +def dns_fatal(hostname) -> bool: + """We have a dns error, should we fail immediately or not? Not for a host that worked before (the error may be + temporary).""" + return hostname not in previously_seen_hostnames class HostRateLimiter: @@ -169,7 +169,7 @@ def myrequests_get( or ('nodename nor servname provided, or not known' in string) # macos or ('getaddrinfo failed' in string) ): # windows - if dns_fatal(url): + if dns_fatal(hostname): raise ValueError('invalid hostname in url ' + url) from None if connect_errors > raise_error_after_n_errors: diff --git a/tests/unit/test_myrequests.py b/tests/unit/test_myrequests.py index 3b3e5b1..8ca7d8f 100644 --- a/tests/unit/test_myrequests.py +++ b/tests/unit/test_myrequests.py @@ -54,3 +54,17 @@ def test_finish_spaces_the_next_request_from_the_end_of_the_previous_one(): with patch('cdx_toolkit.myrequests.time.time', return_value=2000.0): limiter.finish('example.org') assert limiter.reserve('example.org', 2001.0) == 2010.0 + + +@pytest.mark.parametrize('host, fatal', [('unknown-host.example.org', True), ('data.commoncrawl.org', False)]) +def test_dns_error_is_only_fatal_for_unknown_hosts(monkeypatch, host, fatal): + from cdx_toolkit import myrequests + + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, host, 0) + error = requests.exceptions.ConnectionError('[Errno 8] nodename nor servname provided, or not known') + with patch('requests.get', side_effect=error) as get, pytest.raises(ValueError) as e: + myrequests_get(f'https://{host}/x', retry_max_sec=0.01, raise_error_after_n_errors=1) + if fatal: + assert 'invalid hostname' in str(e.value) and get.call_count == 1 + else: + assert '2 failures' in str(e.value) and get.call_count == 2 # retried From 4c2d61e271bfdd5255734489c056cc783cefcf58 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 20:39:57 +0200 Subject: [PATCH 49/91] =?UTF-8?q?refactor(repackage):=20WARCFilter=20in=20?= =?UTF-8?q?pipeline.py=20=E2=86=92=20Repackager=20in=20repackager.py;=20it?= =?UTF-8?q?=20copies=20records=20and=20does=20not=20filter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cdx_toolkit/cli/repackage.py | 6 +- cdx_toolkit/repackage/multiprocess.py | 6 +- .../repackage/{pipeline.py => repackager.py} | 22 ++-- docs/notes/warc-fetcher-performance.md | 37 +++--- tests/repackage/test_pipeline.py | 29 ----- tests/repackage/test_producer.py | 8 +- ..._filter_pipeline.py => test_repackager.py} | 108 ++++++++++++------ tests/unit/test_aws.py | 6 +- tests/unit/test_cli.py | 6 +- 9 files changed, 117 insertions(+), 111 deletions(-) rename cdx_toolkit/repackage/{pipeline.py => repackager.py} (96%) delete mode 100644 tests/repackage/test_pipeline.py rename tests/repackage/{test_warc_filter_pipeline.py => test_repackager.py} (84%) diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index 82e1d80..a45414c 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -213,9 +213,9 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): max_file_size=args.size, ) - from cdx_toolkit.repackage.pipeline import WARCFilter + from cdx_toolkit.repackage.repackager import Repackager - return WARCFilter( + return Repackager( source=source, prefix_path=args.prefix, writer_info=info, @@ -224,7 +224,7 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): log_every_n=args.log_every_n, n_parallel_readers=args.parallel_readers, max_file_size=args.size, - ).filter() + ).run() except MissingWarcLocation as e: raise UsageError(str(e)) diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index bf54c15..1e60474 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -38,7 +38,7 @@ from cdx_toolkit.aws import configure_fsspec_s3 from cdx_toolkit.record_io import RecordWriter, read_records from cdx_toolkit.repackage.jobs import RecordsSource, RepackageResult -from cdx_toolkit.repackage.pipeline import WARCFilter +from cdx_toolkit.repackage.repackager import Repackager from cdx_toolkit.repackage.urls import is_s3_url, parse_s3_uri from cdx_toolkit.repackage.writers.series import generate_warc_filename @@ -80,7 +80,7 @@ def _run_worker(cfg: Dict) -> RepackageResult: os.environ['CDXT_UVLOOP'] = '1' # the shard is in the order of the source (sorted, unless --no-sort), streamed source = RecordsSource(read_records(cfg['shard'], 'jsonl'), warc_download_prefix=None, sort=False) - wf = WARCFilter( + repackager = Repackager( source=source, prefix_path=cfg['prefix_path'], writer_info=cfg['writer_info'], @@ -93,7 +93,7 @@ def _run_worker(cfg: Dict) -> RepackageResult: max_file_size=cfg['max_file_size'], # rotate at --size next_file_sequence=_next_file_sequence, ) - return wf.filter() + return repackager.run() def _split_source_to_shards(source, n: int, out_dir: str) -> Tuple[List[str], List[int]]: diff --git a/cdx_toolkit/repackage/pipeline.py b/cdx_toolkit/repackage/repackager.py similarity index 96% rename from cdx_toolkit/repackage/pipeline.py rename to cdx_toolkit/repackage/repackager.py index e7820cc..fbb7b01 100644 --- a/cdx_toolkit/repackage/pipeline.py +++ b/cdx_toolkit/repackage/repackager.py @@ -57,7 +57,7 @@ def summary(self) -> str: ) -class WARCFilter: +class Repackager: """Copy WARC records, byte for byte, from the WARC files of the range jobs into a series of WARC files. A pipeline of asyncio tasks: @@ -88,11 +88,11 @@ def __init__( max_file_size: Optional[int] = 1 * 1024 * 1024 * 1024, # 1 GiB (for WARC outputs) next_file_sequence: Optional[Callable[[int], int]] = None, ): - """Initialize the WARC filter. + """Set up the pipeline. Args: source: The WARC ranges to repackage: an object with iter_range_jobs() (e.g. a RecordsSource). - prefix_path: Output path prefix for filtered WARC files. + prefix_path: Output path prefix for the WARC files. writer_info: Dictionary containing writer metadata. writer_subprefix: Optional subprefix for writer output paths. write_paths_as_metadata_records: Optional list of file paths to write as metadata records. @@ -137,8 +137,8 @@ def __init__( self.max_file_size = max_file_size self.next_file_sequence = next_file_sequence # see WarcFileSeries - def filter(self) -> RepackageResult: - """Perform the filtering process (calls async method via asyncio.run). + def run(self) -> RepackageResult: + """Run the pipeline (runs run_async in an event loop). Captures that cannot be fetched are counted as failed (the others are still written); a failure to write raises. @@ -152,7 +152,7 @@ def filter(self) -> RepackageResult: logger.info('Using uvloop event loop') except ImportError: logger.warning('CDXT_UVLOOP=1 but uvloop is not installed; using default asyncio loop') - return runner(self.filter_async()) + return runner(self.run_async()) def s3_read_config(self): """botocore config of the S3 client for range reads: high throughput.""" @@ -174,8 +174,8 @@ def s3_write_config(self): connect_timeout=10, ) - async def filter_async(self) -> RepackageResult: - """Filter process using a three stage approach (job generator, warc reader, warc writer).""" + async def run_async(self) -> RepackageResult: + """Run the three stages: producer of range jobs, reader tasks, writer task.""" range_jobs_queue: asyncio.Queue = asyncio.Queue(maxsize=self.range_jobs_queue_size) warc_records_queue: asyncio.Queue = asyncio.Queue(maxsize=self.warc_records_queue_size) @@ -187,7 +187,7 @@ async def filter_async(self) -> RepackageResult: client = aioboto3.Session().client('s3', config=self.s3_write_config()) write_s3_client = await stack.enter_async_context(client) - return await self._run_filter_pipeline( + return await self._run_pipeline( range_jobs_queue=range_jobs_queue, warc_records_queue=warc_records_queue, readers=Readers(stack, self.s3_read_config, self.max_attempts, self.base_backoff_seconds), @@ -225,14 +225,14 @@ def drain() -> int: logger.info('Generated %d range jobs', count) return count - async def _run_filter_pipeline( + async def _run_pipeline( self, range_jobs_queue: asyncio.Queue, warc_records_queue: asyncio.Queue, readers: Readers, write_s3_client=None, ) -> RepackageResult: - """Run the actual filter pipeline with or without S3 client. + """Run the pipeline, with or without an S3 client for the output. A failing writer (or source) stops the whole pipeline, as does cancellation: all tasks are cancelled, and the writer aborts the file it is writing (no incomplete WARC file or S3 object is left). diff --git a/docs/notes/warc-fetcher-performance.md b/docs/notes/warc-fetcher-performance.md index bb64d86..beecf41 100644 --- a/docs/notes/warc-fetcher-performance.md +++ b/docs/notes/warc-fetcher-performance.md @@ -17,27 +17,27 @@ Status legend: **[done]** implemented on this branch · **[proposed]** not yet i ## How the fetcher works today -Three async stages on a single `asyncio` event loop (`cdx_toolkit/repackage/warc_filter.py`): +Three async stages on a single `asyncio` event loop (`cdx_toolkit/repackage/repackager.py`): 1. **Produce** — the source (`RecordsSource`, the captures of the index file) yields `RangeJob`s (one per WARC record: url + byte offset + length). The source is a *sync* generator run in a worker - thread (`asyncio.to_thread(drain)`, `warc_filter.py:~276`), pushing each job onto a + thread (`asyncio.to_thread(drain)`, `repackager.py`), pushing each job onto a bounded `range_jobs_queue` via `run_coroutine_threadsafe(...).result()`. 2. **Read** — `num_readers` reader coroutines (`= --parallel`) each pull a job and do **one S3 `GetObject` (or HTTP GET) per job** (`read_warc_records` → - `RangeJob.ranged_get_bytes`, `warc_filter.py:~446`, `data_classes.py:64`). The + `RangeReader.read`, `readers/`). The payload (raw, already-gzipped WARC record bytes) is enqueued. 3. **Write** — `num_writers` (`= num_readers/6`) writer coroutines each own one output - shard and append records to it (`write_warc_records`, `warc_filter.py:~497`). + shard and append records to it (`write_warc_records` → `WarcFileSeries.write`). Clients/config: - One shared `aioboto3` S3 read client and one write client (`get_aws_clients`, - `warc_filter.py:~179`); read pool sized `num_readers*3`. -- Retries/backoff for throttling (503 SlowDown) in `with_retries` (`s3_utils.py:27`). -- Output writers: `S3ShardWriter` (multipart upload, ≥5 MiB parts) or `LocalFileWriter` + `repackager.py`); read pool sized `num_readers*3`. +- Retries/backoff for throttling (503 SlowDown) in `with_retries` (`retries.py`). +- Output writers: `S3OutputFile` (multipart upload, ≥5 MiB parts) or `LocalOutputFile` (`aiofiles`); shard files keyed by `writer_id` + `sequence` - (`generate_warc_filename`, `warc_utils.py:67`). Records are written **verbatim** — no + (`generate_warc_filename`, `writers/series.py`). Records are written **verbatim** — no recompression. --- @@ -51,8 +51,7 @@ Clients/config: - But asyncio is single-threaded: **one event loop, one OS thread, one CPU core.** All the *CPU* work — TLS encrypt/decrypt, HTTP framing, aiobotocore object churn, byte copies — is serialized on that one core. The readers **and** writers are all - coroutines on the same loop (`asyncio.create_task(...)`, `warc_filter.py:~321` and - `~334`). + coroutines on the same loop (`asyncio.create_task(...)`, `repackager.py`). - The GIL means *threads* wouldn't help CPU-bound work either. Only **multiple processes** use multiple cores. @@ -95,11 +94,11 @@ Run a sample and watch `htop`/`mpstat` during the fetch: ## Writer review (why more output shards won't fix a read-bound core) - **No recompression** — records arrive already gzipped and are written verbatim - (`writer.write(item.data)`, `warc_filter.py:~577`). The writer adds no gzip CPU. -- **`S3ShardWriter`** (`s3_writer.py:84`) buffers into a `bytearray`; on each ≥5 MiB + (`writer.write(item.data)`, `repackager.py`). The writer adds no gzip CPU. +- **`S3OutputFile`** (`writers/s3.py`) buffers into a `bytearray`; on each ≥5 MiB boundary it does `buffer[:n]` (copy), `del buffer[:n]` (an O(remaining) memmove), `bytes(chunk)` (another copy), then `await upload_part`. Real but modest byte-copy cost. -- **`LocalFileWriter`** (`local_writer.py`) is `aiofiles` with an 8 KiB buffer and a +- **`LocalOutputFile`** (`writers/local.py`) is `aiofiles` with an 8 KiB buffer and a `flush()` per write — on a fast read stream the per-flush overhead + EBS limits are the more likely bottleneck (prefer writing to `s3://` on a c5n). - **[done] One writer per process.** Because writing is a cheap verbatim copy that one @@ -150,12 +149,12 @@ Assume parallelism is already configured (`--parallel` high) and reads go to process's producer thread (CSV sort + per-row cross-thread enqueue) competes with its event loop for the GIL — see latent issues. -4. **[done] uvloop.** Gated by `CDXT_UVLOOP=1` in `WARCFilter.filter` (`warc_filter.py`). +4. **[done] uvloop.** Gated by `CDXT_UVLOOP=1` in `Repackager.run` (`repackager.py`). Measured ~+8 % single-core; stacks under multi-process. 5. **[proposed] Bound coalesced read size / watch RAM.** A c5n.xlarge has 10.5 GiB; whole-file (~1 GB) reads × many readers will OOM. Cap superrange size and gap - threshold; revisit `warc_records_queue_size` (`warc_filter.py:~62`, default 200) since + threshold; revisit `warc_records_queue_size` (`repackager.py`, default 200) since each buffered payload is larger after coalescing. 6. **[done] Write output to `s3://` in-region, not local EBS.** Measured: S3-direct MPU @@ -176,14 +175,14 @@ Assume parallelism is already configured (`--parallel` high) and reads go to ### Latent issues worth noting -- **HTTP path blocks the event loop.** `ranged_get_bytes`'s HTTP branch calls the - *synchronous* `myrequests_get(...)` without `await`/`to_thread` (`data_classes.py:~91`). +- **HTTP path blocks the event loop.** The HTTP read (then in `RangeJob.ranged_get_bytes`) called the + *synchronous* `myrequests_get(...)` without `await`/`to_thread`. Since the default `--warc-download-prefix` is `https://data.commoncrawl.org`, the default fetch path serializes the "parallel" readers. On EC2, prefer `s3://commoncrawl` (the S3 branch is properly async); otherwise the HTTP fetch should be offloaded to a - thread pool / async HTTP client. **[proposed]** + thread pool / async HTTP client. **[done]**: `HttpRangeReader` (`readers/http.py`) runs it in a thread. - **Per-row cross-thread enqueue.** The producer does one - `run_coroutine_threadsafe(...).result()` per job (`warc_filter.py:~268`) — 1M + `run_coroutine_threadsafe(...).result()` per job (`repackager.py`) — 1M loop↔thread handoffs. Batching enqueues removes a real per-row overhead at this scale. **[proposed]** diff --git a/tests/repackage/test_pipeline.py b/tests/repackage/test_pipeline.py deleted file mode 100644 index e7f01c4..0000000 --- a/tests/repackage/test_pipeline.py +++ /dev/null @@ -1,29 +0,0 @@ -from tests.conftest import TEST_DATA_PATH - -from cdx_toolkit.repackage.pipeline import ThroughputTracker, WARCFilter -from cdx_toolkit.repackage.jobs import RecordsSource - -fixture_path = TEST_DATA_PATH / 'repackage' - -# A throwaway source for unit tests that only exercise reader/writer/rotate/log methods. -_FAKE_SOURCE = RecordsSource([], 'https://data.commoncrawl.org') - - -def test_log_progress(caplog): - """Progress is logged every --log-every-n items.""" - import logging - caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.pipeline') - - warc_filter = WARCFilter( - source=_FAKE_SOURCE, - prefix_path='/fake/prefix', - writer_info={'writer_id': 1}, - log_every_n=2, - ) - tracker = ThroughputTracker() - for _ in range(5): - tracker.add(bytes_count=1024 * 1024, records_count=1) - warc_filter.log_progress('WARC Reader 1', tracker) - - assert caplog.text.count('WARC Reader 1: ') == 2 - assert 'WARC Reader 1: 4 items, 4.0 MB in ' in caplog.text diff --git a/tests/repackage/test_producer.py b/tests/repackage/test_producer.py index 607b660..46e02e8 100644 --- a/tests/repackage/test_producer.py +++ b/tests/repackage/test_producer.py @@ -4,7 +4,7 @@ import pytest from cdx_toolkit.repackage.jobs import RangeJob -from cdx_toolkit.repackage.pipeline import WARCFilter +from cdx_toolkit.repackage.repackager import Repackager from cdx_toolkit.repackage.readers.http import HttpRangeReader @@ -19,7 +19,9 @@ def test_failing_source_fails_the_pipeline(tmp_path): async def read(self, job): return b'x' - wf = WARCFilter(source=FailingSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=3) + repackager = Repackager( + source=FailingSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=3, + ) with patch.object(HttpRangeReader, 'read', read), pytest.raises(RuntimeError, match='boom'): - wf.filter() + repackager.run() assert glob.glob(str(tmp_path / 'out-*')) == [] diff --git a/tests/repackage/test_warc_filter_pipeline.py b/tests/repackage/test_repackager.py similarity index 84% rename from tests/repackage/test_warc_filter_pipeline.py rename to tests/repackage/test_repackager.py index 2f845e5..522cee6 100644 --- a/tests/repackage/test_warc_filter_pipeline.py +++ b/tests/repackage/test_repackager.py @@ -1,10 +1,11 @@ -"""The WARC fetch/write pipeline end to end without network: fake HTTP and HF reads, in-process "processes".""" +"""The Repackager pipeline end to end without network: fake HTTP and HF reads, in-process "processes".""" import asyncio import contextlib import glob import io import gzip import itertools +import logging import os import threading import time @@ -22,7 +23,7 @@ from cdx_toolkit.repackage.readers.s3 import S3RangeReader from cdx_toolkit.repackage.writers.local import LocalOutputFile from cdx_toolkit.repackage.jobs import RecordsSource -from cdx_toolkit.repackage.pipeline import WARCFilter +from cdx_toolkit.repackage.repackager import Repackager, ThroughputTracker from tests.conftest import InlineExecutor from tests.fake_warc_server import FakeWarcServer from tests.conftest import requires_aioboto3 @@ -58,10 +59,11 @@ def flaky_get(url, headers=None, **kwargs): raise ConnectionError('network down') return fake_get(url, headers) - wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), - writer_info={}) + repackager = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + ) with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=flaky_get): - assert wf.filter() == RepackageResult(requested=4, written=3, failed=1) + assert repackager.run() == RepackageResult(requested=4, written=3, failed=1) assert len(response_records(glob.glob(str(tmp_path / 'out-*.warc.gz')))) == 3 @@ -77,10 +79,11 @@ def test_invalid_range_responses_are_not_written(tmp_path, response): if response == 'wrong-range': response = Mock(status_code=206, headers={'Content-Range': f'bytes {offset + 1}-{offset + length}/9999'}, content=b'x' * length) - wf = WARCFilter(source=RecordsSource([capture], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), - writer_info={}) + repackager = Repackager( + source=RecordsSource([capture], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + ) with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=response): - assert wf.filter() == RepackageResult(requested=1, written=0, failed=1) + assert repackager.run() == RepackageResult(requested=1, written=0, failed=1) assert glob.glob(str(tmp_path / 'out-*')) == [] @@ -99,18 +102,22 @@ async def write(self, data): if b'https://example.com/1' in gzip.decompress(data): second_written.set() - wf = WARCFilter(source=RecordsSource(CAPTURES[:2], 'https://data.example.org', sort=False), - prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=2) + repackager = Repackager( + source=RecordsSource(CAPTURES[:2], 'https://data.example.org', sort=False), prefix_path=str(tmp_path / 'out'), + writer_info={}, n_parallel_readers=2, + ) with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=get), \ patch.object(LocalOutputFile, 'write', write): - assert wf.filter() == RepackageResult(requested=2, written=2) + assert repackager.run() == RepackageResult(requested=2, written=2) records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert [r.rec_headers.get_header('WARC-Target-URI') for r in records] == [CAPTURES[1]['url'], CAPTURES[0]['url']] def test_writer_error_stops_the_pipeline_and_removes_the_file(tmp_path): - wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), - writer_info={}, max_file_size=1) + repackager = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + max_file_size=1, + ) writes = [] original_write = LocalOutputFile.write @@ -123,7 +130,7 @@ async def failing_write(self, data): with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ patch.object(LocalOutputFile, 'write', failing_write), pytest.raises(OSError, match='disk full'): - wf.filter() + repackager.run() # the first file is complete, the second was removed assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == ['out-001.warc.gz'] @@ -133,8 +140,10 @@ async def failing_write(self, data): (2, ['out-001.warc.gz']), # the final close: the last file is removed, the complete first one stays ]) def test_close_error_stops_the_pipeline_and_removes_the_file(tmp_path, failing_close, expected_files): - wf = WARCFilter(source=RecordsSource(CAPTURES[:2], 'https://data.example.org'), - prefix_path=str(tmp_path / 'out'), writer_info={}, max_file_size=1) + repackager = Repackager( + source=RecordsSource(CAPTURES[:2], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}, max_file_size=1, + ) flushes = [] original_flush = LocalOutputFile._flush @@ -146,15 +155,17 @@ async def failing_flush(self): with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ patch.object(LocalOutputFile, '_flush', failing_flush), pytest.raises(OSError, match='disk full'): - wf.filter() + repackager.run() assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == expected_files def test_rotation_by_size(tmp_path): - wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), - writer_info={}, max_file_size=1) + repackager = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + max_file_size=1, + ) with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): - assert wf.filter().written == 4 + assert repackager.run().written == 4 paths = sorted(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert [os.path.basename(p) for p in paths] == [f'out-00{i}.warc.gz' for i in range(1, 5)] assert all(len(response_records([p])) == 1 for p in paths) @@ -180,11 +191,12 @@ async def blocking_read(self, job): await asyncio.Event().wait() # blocks forever return b'x' - wf = WARCFilter(source=EndlessSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, - range_jobs_queue_size=2) + repackager = Repackager( + source=EndlessSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, range_jobs_queue_size=2, + ) async def run(): - task = asyncio.create_task(wf.filter_async()) + task = asyncio.create_task(repackager.run_async()) while len(reads) < 2: await asyncio.sleep(0.01) await asyncio.sleep(0.2) # the queue fills up, the producer waits @@ -264,10 +276,12 @@ def get(*args, **kwargs): def test_hf_reads_use_the_hf_reader(tmp_path): - wf = WARCFilter(source=RecordsSource(CAPTURES, 'hf://buckets/commoncrawl/commoncrawl'), - prefix_path=str(tmp_path / 'out'), writer_info={}) + repackager = Repackager( + source=RecordsSource(CAPTURES, 'hf://buckets/commoncrawl/commoncrawl'), prefix_path=str(tmp_path / 'out'), + writer_info={}, + ) with patch('cdx_toolkit.repackage.readers.hf.make_hf_reader', fake_make_hf_reader): - assert wf.filter().written == 4 + assert repackager.run().written == 4 records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) assert sorted(r.rec_headers.get_header('WARC-Target-URI') for r in records) == sorted(c['url'] for c in CAPTURES) @@ -277,10 +291,10 @@ def test_uvloop(monkeypatch, caplog, uvloop_installed): monkeypatch.setenv('CDXT_UVLOOP', '1') fake_uvloop = Mock(run=asyncio.run) if uvloop_installed else None monkeypatch.setitem(__import__('sys').modules, 'uvloop', fake_uvloop) - wf = WARCFilter(source=RecordsSource([], None), prefix_path='/nonexistent/out', writer_info={}) + repackager = Repackager(source=RecordsSource([], None), prefix_path='/nonexistent/out', writer_info={}) with caplog.at_level('INFO'): - assert wf.filter() == RepackageResult() + assert repackager.run() == RepackageResult() assert ('Using uvloop event loop' if uvloop_installed else 'uvloop is not installed') in caplog.text @@ -351,7 +365,7 @@ def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor): session.return_value.client.return_value = FakeS3Client() if processes == 1: - result = WARCFilter(source=source, prefix_path=str(tmp_path / 'out'), writer_info={}).filter() + result = Repackager(source=source, prefix_path=str(tmp_path / 'out'), writer_info={}).run() else: result = multiprocess.run_multiprocess_repackage( source=source, n_processes=processes, readers_per_process=1, prefix_path=str(tmp_path / 'out'), @@ -372,8 +386,10 @@ def test_s3_output_with_metadata_records(tmp_path): with patch('aioboto3.Session') as session, \ patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): session.return_value.client.return_value = s3 - result = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path='s3://bucket/out', - writer_info={}, write_paths_as_metadata_records=[str(metadata)]).filter() + result = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path='s3://bucket/out', writer_info={}, + write_paths_as_metadata_records=[str(metadata)], + ).run() assert result.written == 4 data = s3.objects['s3://bucket/out-001.warc.gz'] records = [(r.rec_type, r.content_stream().read()) for r in ArchiveIterator(io.BytesIO(data))] @@ -382,11 +398,13 @@ def test_s3_output_with_metadata_records(tmp_path): def test_failing_metadata_record_removes_the_file(tmp_path): - wf = WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), - writer_info={}, write_paths_as_metadata_records=[str(tmp_path / 'missing.txt')]) + repackager = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + write_paths_as_metadata_records=[str(tmp_path / 'missing.txt')], + ) with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ pytest.raises(FileNotFoundError): - wf.filter() + repackager.run() assert glob.glob(str(tmp_path / 'out-*')) == [] @@ -447,8 +465,10 @@ def test_s3_reads_retry_the_whole_transfer(): def test_no_s3_client_without_s3(tmp_path): with patch('aioboto3.Session') as session, \ patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): - WARCFilter(source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), - writer_info={}).filter() + Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}, + ).run() session.assert_not_called() @@ -499,4 +519,18 @@ def test_multiprocess_repackage_keeps_the_order_of_the_source(tmp_path): def test_at_least_one_reader(): with pytest.raises(ValueError, match='n_parallel_readers'): - WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=0) + Repackager(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=0) + + +def test_log_progress(caplog): + """Progress is logged every --log-every-n items.""" + caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.repackager') + + repackager = Repackager(source=RecordsSource([], None), prefix_path='/fake/prefix', writer_info={}, log_every_n=2) + tracker = ThroughputTracker() + for _ in range(5): + tracker.add(bytes_count=1024 * 1024, records_count=1) + repackager.log_progress('WARC Reader 1', tracker) + + assert caplog.text.count('WARC Reader 1: ') == 2 + assert 'WARC Reader 1: 4 items, 4.0 MB in ' in caplog.text diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py index 1a82f50..13fce18 100644 --- a/tests/unit/test_aws.py +++ b/tests/unit/test_aws.py @@ -131,10 +131,10 @@ def test_s3fs_uses_the_defaults(no_user_config, monkeypatch): @requires_aioboto3 def test_warc_fetcher_s3_clients(no_user_config): from cdx_toolkit.repackage.jobs import RecordsSource - from cdx_toolkit.repackage.pipeline import WARCFilter + from cdx_toolkit.repackage.repackager import Repackager - wf = WARCFilter(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=4) - read_config, write_config = wf.s3_read_config(), wf.s3_write_config() + repackager = Repackager(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=4) + read_config, write_config = repackager.s3_read_config(), repackager.s3_write_config() for config in (read_config, write_config): assert config.region_name == 'us-east-1' assert config.retries == {'mode': 'adaptive', 'total_max_attempts': 100000} diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index ba1dffa..a8f855c 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -152,15 +152,15 @@ def test_index_to_parquet_and_repackage(tmp_path, capsys): def test_repackage_options_are_passed_to_the_warc_fetcher(tmp_path): captures = _write_captures(tmp_path / 'captures.jsonl', []) - with patch('cdx_toolkit.repackage.pipeline.WARCFilter') as warc_filter: - warc_filter.return_value.filter.return_value = RepackageResult() + with patch('cdx_toolkit.repackage.repackager.Repackager') as repackager: + repackager.return_value.run.return_value = RepackageResult() main([ 'repackage', captures, '--prefix', 'FOO', '--subprefix', 'BAR', '--size', '1MiB', '--creator', 'me', '--description', 'test', '--add-metadata-record', 'a.txt', '--parallel-readers', '4', '--warc-download-prefix', 's3://commoncrawl', ]) - kwargs = warc_filter.call_args.kwargs + kwargs = repackager.call_args.kwargs assert kwargs['prefix_path'] == 'FOO' assert kwargs['writer_subprefix'] == 'BAR' assert kwargs['max_file_size'] == 1024 * 1024 From 33f4f7ad2d3cd7c35dcca819f16ad8f06addd408 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 21:32:25 +0200 Subject: [PATCH 50/91] refactor(indexes): open_index(); records(), count() and explain() validate the query Backends implement select() and select_count(); a single string is accepted for the list fields of IndexQuery. --- cdx_toolkit/cli/index.py | 7 ++--- cdx_toolkit/indexes/__init__.py | 2 +- cdx_toolkit/indexes/backends/cdx_api.py | 2 +- cdx_toolkit/indexes/backends/cdx_files.py | 3 ++- cdx_toolkit/indexes/backends/sql.py | 2 +- cdx_toolkit/indexes/base.py | 20 +++++++++++---- cdx_toolkit/indexes/query.py | 9 +++++++ cdx_toolkit/indexes/resolve.py | 2 +- tests/unit/test_cli.py | 4 +-- tests/unit/test_indexes.py | 31 +++++++++++++++++++---- tests/unit/test_sql_indexes.py | 6 ++--- 11 files changed, 63 insertions(+), 25 deletions(-) diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index f295e68..107e660 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -1,7 +1,7 @@ from argparse import Namespace, RawDescriptionHelpFormatter import logging -from cdx_toolkit.indexes import IndexQuery, UsageError, resolve_index +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index from cdx_toolkit.indexes.resolve import INDEX_FORMATS from cdx_toolkit.record_io import ( FORMAT_CHOICES, canonical_format, format_from_path, import_pyarrow, open_text, write_records, @@ -220,11 +220,10 @@ def writes_per_file(args: Namespace, backend) -> bool: def run(args: Namespace, cmdline: str): - backend = resolve_index( + backend = open_index( args.index, args.index_format, cc_mirror=args.cc_mirror, cc_sort=args.cc_sort, assume_yes=args.yes ) query = build_query(args) - backend.validate(query) if writes_per_file(args, backend): if args.fields: @@ -243,8 +242,6 @@ def run(args: Namespace, cmdline: str): return if args.count: - if query.url_fgrep or query.url_fgrepv: - raise UsageError('--count cannot be used with --url-fgrep or --url-fgrepv') count = backend.count(query) with open_text(args.output, 'w') as fh: fh.write(f'{count}\n') diff --git a/cdx_toolkit/indexes/__init__.py b/cdx_toolkit/indexes/__init__.py index f3bb730..9040ef7 100644 --- a/cdx_toolkit/indexes/__init__.py +++ b/cdx_toolkit/indexes/__init__.py @@ -1,3 +1,3 @@ from .base import IndexBackend, QueryPlan, UsageError # noqa: F401 from .query import IndexQuery # noqa: F401 -from .resolve import resolve_index # noqa: F401 +from .resolve import open_index # noqa: F401 diff --git a/cdx_toolkit/indexes/backends/cdx_api.py b/cdx_toolkit/indexes/backends/cdx_api.py index df9c3a2..9aafce9 100644 --- a/cdx_toolkit/indexes/backends/cdx_api.py +++ b/cdx_toolkit/indexes/backends/cdx_api.py @@ -54,7 +54,7 @@ def select(self, query: IndexQuery) -> Iterator[Mapping]: seen.add(identity) yield capture - def count(self, query: IndexQuery) -> int: + def select_count(self, query: IndexQuery) -> int: """Estimate from the number of index pages, at most --limit; overlapping selectors are counted more than once.""" cdx = self._fetcher(query) diff --git a/cdx_toolkit/indexes/backends/cdx_files.py b/cdx_toolkit/indexes/backends/cdx_files.py index 8697d6c..4706ab9 100644 --- a/cdx_toolkit/indexes/backends/cdx_files.py +++ b/cdx_toolkit/indexes/backends/cdx_files.py @@ -236,7 +236,7 @@ def select(self, query: IndexQuery) -> Iterator[Mapping]: for line in iter_matching_lines(cdx_file.path, matcher): yield parse_cdxj_line(line) - def count(self, query: IndexQuery) -> int: + def select_count(self, query: IndexQuery) -> int: """Exact number of selected captures, at most --limit (reads all files, or until the limit).""" return sum(1 for _ in islice(self.select(query), query.limit)) @@ -244,6 +244,7 @@ def write_per_file(self, query: IndexQuery, output_base: str, processes: int = 1 """Write the selected lines of each CDX file, unchanged, to /. Returns the number of output files and captures; no file is written without matches.""" + self.validate(query) if query.limit: raise UsageError('--limit cannot be used when writing one output per CDX file') capture_filter = CaptureFilter.from_query(query, url_filters=True) diff --git a/cdx_toolkit/indexes/backends/sql.py b/cdx_toolkit/indexes/backends/sql.py index f62edc3..e4372fc 100644 --- a/cdx_toolkit/indexes/backends/sql.py +++ b/cdx_toolkit/indexes/backends/sql.py @@ -156,7 +156,7 @@ def select(self, query: IndexQuery) -> Iterator[Mapping]: for row in self.iter_sql_rows(plan.sql): yield normalize_row(row) - def count(self, query: IndexQuery) -> int: + def select_count(self, query: IndexQuery) -> int: plan = self.plan(query, order_by=False) confirm_cost(plan.n_crawls, self.engine, self.assume_yes) row = next(self.iter_sql_rows(f'SELECT count(*) AS n FROM ({plan.sql}) AS q')) diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py index dcb773c..f5de51f 100644 --- a/cdx_toolkit/indexes/base.py +++ b/cdx_toolkit/indexes/base.py @@ -10,7 +10,7 @@ class UsageError(ValueError): - """Invalid combination of command line options, reported without a traceback.""" + """Invalid query or options (the command line reports it without a traceback).""" class QueryPlan(ABC): @@ -25,7 +25,8 @@ class IndexBackend(ABC): """An index that can be queried for captures: the CDX API, CDX files, or the columnar index (Athena, DuckDB). A query (IndexQuery, the same for every backend) becomes the backend's plan (plan()), which is run by - select() and count(), or shown by explain().""" + select() and select_count(), or shown by explain(). The public methods records(), count() and explain() + validate the query first.""" name: str = '' supported_options: FrozenSet[str] = frozenset() @@ -53,17 +54,26 @@ def select(self, query: IndexQuery) -> Iterator[Mapping]: raise NotImplementedError # pragma: no cover @abstractmethod - def count(self, query: IndexQuery) -> int: - """Cheap (possibly estimated) number of captures matching the query, at most --limit (without the - client-side --url-fgrep/--url-fgrepv, which cannot be combined with --count).""" + def select_count(self, query: IndexQuery) -> int: + """Cheap (possibly estimated) number of captures matching the query, at most --limit (see count()).""" raise NotImplementedError # pragma: no cover def records(self, query: IndexQuery) -> Iterator[Mapping]: """The captures of the query: at most --limit, then filtered by --url-fgrep/--url-fgrepv (client-side).""" + self.validate(query) return filter_by_url(islice(self.select(query), query.limit), query.url_fgrep, query.url_fgrepv) + def count(self, query: IndexQuery) -> int: + """The number of captures matching the query, at most --limit; without the client-side + --url-fgrep/--url-fgrepv, which cannot be combined with it.""" + self.validate(query) + if query.url_fgrep or query.url_fgrepv: + raise UsageError('--count cannot be used with --url-fgrep or --url-fgrepv') + return self.select_count(query) + def explain(self, query: IndexQuery) -> str: """Describe what would be queried (--dry-run), without querying the index.""" + self.validate(query) return self.plan(query).describe() diff --git a/cdx_toolkit/indexes/query.py b/cdx_toolkit/indexes/query.py index 012e8f7..c983dd7 100644 --- a/cdx_toolkit/indexes/query.py +++ b/cdx_toolkit/indexes/query.py @@ -13,6 +13,9 @@ } +LIST_FIELDS = ('urls', 'hostnames', 'domains', 'filters', 'languages') + + def field_flag(name: str) -> str: return FIELD_FLAGS.get(name, '--' + name.replace('_', '-')) @@ -42,6 +45,12 @@ class IndexQuery: languages: List[str] = field(default_factory=list) sql: Optional[str] = None + def __post_init__(self): + # a single value, e.g. IndexQuery(urls='example.com/*'), would otherwise be iterated by character + for name in LIST_FIELDS: + if isinstance(getattr(self, name), str): + setattr(self, name, [getattr(self, name)]) + def url_patterns(self) -> List[Tuple[str, Optional[str]]]: """All URL selectors as (pattern, CDX API matchType) pairs.""" return ( diff --git a/cdx_toolkit/indexes/resolve.py b/cdx_toolkit/indexes/resolve.py index 1eb17dd..7e24fd7 100644 --- a/cdx_toolkit/indexes/resolve.py +++ b/cdx_toolkit/indexes/resolve.py @@ -41,7 +41,7 @@ def detect_index_format(location: str) -> str: return 'cdx' -def resolve_index( +def open_index( location: str, index_format: str = 'auto', cc_mirror: Optional[str] = None, diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index a8f855c..4ad696e 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -136,8 +136,8 @@ def test_index_to_parquet_and_repackage(tmp_path, capsys): pytest.importorskip('pyarrow') captures = tmp_path / 'captures.parquet' - with patch.object(cdx_toolkit.cli.index, 'resolve_index') as resolve_index: - resolve_index.return_value.records.return_value = iter([CC_CAPTURE]) + with patch.object(cdx_toolkit.cli.index, 'open_index') as open_index: + open_index.return_value.records.return_value = iter([CC_CAPTURE]) main(['index', 'cc', '--url', 'example.com/*', '-o', str(captures)]) with pytest.raises(SystemExit): main(['index', 'cc', '--url', 'example.com/*', '--output-format', 'parquet']) diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py index 30d54bb..9b12f90 100644 --- a/tests/unit/test_indexes.py +++ b/tests/unit/test_indexes.py @@ -2,7 +2,7 @@ import pytest -from cdx_toolkit.indexes import IndexQuery, UsageError, resolve_index +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend from cdx_toolkit.indexes.resolve import is_cdx_api_location @@ -29,15 +29,15 @@ def test_is_cdx_api_location(location, is_api): assert is_cdx_api_location(location) == is_api -def test_resolve_index(): - backend = resolve_index('cc', cc_mirror='https://mirror.example.com/', cc_sort='ascending') +def test_open_index(): + backend = open_index('cc', cc_mirror='https://mirror.example.com/', cc_sort='ascending') assert isinstance(backend, CdxApiBackend) assert (backend.source, backend.cc_mirror, backend.cc_sort) == ('cc', 'https://mirror.example.com/', 'ascending') - cdx_files = resolve_index('s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz') + cdx_files = open_index('s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz') assert isinstance(cdx_files, CdxFilesBackend) with pytest.raises(UsageError, match='unknown index format'): - resolve_index('cc', index_format='parquet') + open_index('cc', index_format='parquet') def test_query_options_used(): @@ -46,6 +46,27 @@ def test_query_options_used(): assert IndexQuery().options_used() == set() +def test_query_single_values_are_lists(): + query = IndexQuery(urls='example.com/*', domains='example.org', filters='=status:200', languages='eng') + assert (query.urls, query.domains, query.filters, query.languages) == ( + ['example.com/*'], ['example.org'], ['=status:200'], ['eng']) + + +@pytest.mark.parametrize('method', ['records', 'count', 'explain']) +def test_public_methods_validate_the_query(method): + """A Python caller does not need to call validate() first.""" + with patch.object(CdxApiBackend, 'select') as select, patch.object(CdxApiBackend, 'select_count') as count, \ + pytest.raises(UsageError, match='at least one --url'): + getattr(CdxApiBackend('cc'), method)(IndexQuery(crawl='3')) + select.assert_not_called() + count.assert_not_called() + + +def test_count_with_url_fgrep(): + with pytest.raises(UsageError, match='--count cannot be used with --url-fgrep'): + CdxApiBackend('cc').count(IndexQuery(urls='example.com/*', url_fgrep='x')) + + def test_cdx_api_validate(): CdxApiBackend('cc').validate(IndexQuery(domains=['example.com'], crawl='3')) diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index 1587875..dcd97f0 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -5,7 +5,7 @@ import pytest from cdx_toolkit.cli import main -from cdx_toolkit.indexes import IndexQuery, UsageError, resolve_index +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend from cdx_toolkit.indexes.backends.athena import AthenaBackend, AthenaLocation @@ -34,7 +34,7 @@ ], ) def test_resolve_index(location, index_format, backend, backend_location): - resolved = resolve_index(location, index_format) + resolved = open_index(location, index_format) assert isinstance(resolved, backend) if backend_location: assert resolved.location == backend_location @@ -50,7 +50,7 @@ def test_resolve_index(location, index_format, backend, backend_location): ) def test_resolve_index_errors(location, index_format, error): with pytest.raises(UsageError, match=error): - resolve_index(location, index_format) + open_index(location, index_format) def test_athena_location(): From 9976318c232855972e61e7c20d3ec692bfb55d89 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 21:41:36 +0200 Subject: [PATCH 51/91] refactor(repackage): --wb replays through the Repackager and WarcFileSeries WaybackJob/WaybackSource and a WaybackReader (Readers.for_job); --wb files are named -001.warc.gz and --parallel-readers and --add-metadata-record work with --wb. Nothing inside cdx_toolkit uses cdx_toolkit.warc.get_writer anymore. --- CHANGELOG.md | 3 +- README.md | 6 ++- cdx_toolkit/cli/repackage.py | 64 ++++++----------------- cdx_toolkit/repackage/jobs.py | 49 +++++++++++++++-- cdx_toolkit/repackage/multiprocess.py | 24 ++++++--- cdx_toolkit/repackage/readers/__init__.py | 25 +++++---- cdx_toolkit/repackage/readers/wayback.py | 26 +++++++++ cdx_toolkit/repackage/repackager.py | 20 +++---- cdx_toolkit/warc.py | 9 ++-- docs/cli-migration-guide.md | 3 +- tests/repackage/test_readers.py | 26 ++++++--- tests/repackage/test_repackager.py | 28 +++++++++- tests/test_cli.py | 4 +- tests/unit/test_cli.py | 48 +++++++++++------ 14 files changed, 223 insertions(+), 112 deletions(-) create mode 100644 cdx_toolkit/repackage/readers/wayback.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff5eb3..29cada7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ + `cdxt repackage` copies WARC records byte-for-byte with the async WARC fetcher (from the feat/warc-range-sources branch): reads from https://, s3:// and hf:// (Hugging Face Storage Buckets), writes to local, s3:// or hf://, `--processes`, `--parallel-readers`, `--add-metadata-record`, glob input; output files are now named `-001.warc.gz` - (`--wb` wayback replays keep the old writer and names); with `--processes`, the processes share one file counter + (`--wb` wayback replays too, which now work with `--parallel-readers` and `--add-metadata-record`); with + `--processes`, the processes share one file counter and write the final file names directly, so each `warcinfo` record names its own file + `hf://` WARC reads resolve each WARC file once and read its records from the signed CDN URL (one HF Resolvers rate-limit request per WARC file instead of per record) diff --git a/README.md b/README.md index 23f64f3..a4ec8ec 100644 --- a/README.md +++ b/README.md @@ -205,8 +205,10 @@ for benchmarks and tuning. Captures without a WARC location, like Internet Archive captures, are replayed from the wayback machine given with `--wb` (`ia` is a shortcut for -`https://web.archive.org/web`) and written one by one to -`[-]-000000.extracted.warc.gz`, ... +`https://web.archive.org/web`) and written to the same +`[-]-001.warc.gz`, ... series, also with `--parallel-readers` and +`--add-metadata-record` (`--processes` cannot be used with `--wb`: the requests to the wayback machine +are rate-limited per process). In both cases, if WARC files with the same prefix and subprefix exist already (e.g. from an earlier run), `cdxt repackage` stops before fetching (exit diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index a45414c..aa40cf9 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -2,24 +2,19 @@ import logging import os import time -from typing import Mapping, Optional +from typing import Optional -import cdx_toolkit.warc from cdx_toolkit.indexes import UsageError from cdx_toolkit.record_io import ( FORMAT_CHOICES, canonical_format, check_glob, format_from_path, import_pyarrow, read_records, ) from cdx_toolkit.utils import get_version -from cdx_toolkit.warc import fetch_wb_warc from .common import IncompleteError, add_verbose_arg, parse_size, positive_int LOGGER = logging.getLogger(__name__) -# wayback names that map to replay URLs -WAYBACK_NAMES = {'ia': 'https://web.archive.org/web'} - DESCRIPTION = '''\ Fetch the captures listed in an index file (as written by `cdxt index`) and write them to WARC files. @@ -28,9 +23,9 @@ [-]-001.warc.gz, -002, ... (local, s3:// or hf://buckets/...). Captures without a WARC location (e.g. Internet Archive captures) are replayed from a wayback -machine with --wb and written to [-]-000000.extracted.warc.gz, ... +machine with --wb and written to the same series of WARC files. -WARC files with the same prefix and subprefix (of either kind) must not exist yet. +WARC files with the same prefix and subprefix must not exist yet. ''' EXAMPLES = '''\ @@ -155,10 +150,7 @@ def run(args: Namespace, cmdline: str): info = build_warcinfo(args, cmdline) start_time = time.time() - if args.wb: - result = repackage_from_wayback(args, records, info) - else: - result = repackage_by_warc_location(args, records, info) + result = repackage_captures(args, records, info) LOGGER.info('wrote %d of %d captures to WARC files (%d failed) in %.1f seconds', result.written, result.requested, result.failed, time.time() - start_time) @@ -182,14 +174,21 @@ def check_output_is_new(prefix: str, subprefix: Optional[str]) -> None: ) -def repackage_by_warc_location(args: Namespace, records, info: dict): - """Copy the WARC records byte-for-byte with the async WARC fetcher (optionally multi-process).""" +def repackage_captures(args: Namespace, records, info: dict): + """Copy the WARC records byte-for-byte, or replay them from a wayback machine with --wb, with the async + repackager (optionally multi-process).""" import fsspec from cdx_toolkit.repackage.urls import is_hf_url - from cdx_toolkit.repackage.jobs import MissingWarcLocation, RecordsSource + from cdx_toolkit.repackage.jobs import MissingWarcLocation, RecordsSource, WaybackSource try: - source = RecordsSource(records, args.warc_download_prefix, sort=not args.no_sort) + if args.wb: + # the minimum interval between requests to the wayback machine only holds within a process + if args.processes > 1: + raise UsageError('--processes cannot be used with --wb') + source = WaybackSource(records, args.wb) + else: + source = RecordsSource(records, args.warc_download_prefix, sort=not args.no_sort) # make sure the output directory exists prefix_fs, prefix_fs_path = fsspec.url_to_fs(args.prefix) @@ -227,36 +226,3 @@ def repackage_by_warc_location(args: Namespace, records, info: dict): ).run() except MissingWarcLocation as e: raise UsageError(str(e)) - - -def repackage_from_wayback(args: Namespace, records, info: dict): - """Replay the captures from a wayback machine, one by one, and write them with the WARC writer of cdx_toolkit.""" - for option, used in ( - ('--processes', args.processes > 1), - ('--parallel-readers', args.parallel_readers > 1), - ('--add-metadata-record', args.add_metadata_record), - ): - if used: - raise UsageError(f'{option} cannot be used with --wb') - - from cdx_toolkit.repackage.jobs import RepackageResult - - wb = WAYBACK_NAMES.get(args.wb, args.wb) - requested = written = 0 - with cdx_toolkit.warc.get_writer(args.prefix, args.subprefix, info, size=args.size) as writer: - for capture in records: - requested += 1 - try: - record = fetch_record(capture, wb) - except Exception as e: - LOGGER.error('cannot fetch capture %s %s: %r', capture.get('url'), capture.get('timestamp'), e) - continue - writer.write_record(record) - written += 1 - return RepackageResult(requested=requested, written=written, failed=requested - written) - - -def fetch_record(capture: Mapping, wb: Optional[str]): - if capture.get('mime') == 'warc/revisit': - LOGGER.warning('revisit record being resolved for url %s %s', capture['url'], capture['timestamp']) - return fetch_wb_warc(capture, wb=wb) diff --git a/cdx_toolkit/repackage/jobs.py b/cdx_toolkit/repackage/jobs.py index 84c2862..3376bc1 100644 --- a/cdx_toolkit/repackage/jobs.py +++ b/cdx_toolkit/repackage/jobs.py @@ -1,9 +1,10 @@ ''' -The captures to repackage: index records (as written by `cdxt index`) -> range jobs, and the result of a run. +The captures to repackage: index records (as written by `cdxt index`) -> jobs (a WARC range, or a wayback +replay), and the result of a run. ''' import logging -from dataclasses import dataclass -from typing import Iterable, Iterator, Mapping, Optional +from dataclasses import dataclass, field +from typing import Iterable, Iterator, Mapping, Optional, Union logger = logging.getLogger(__name__) @@ -35,12 +36,32 @@ class RangeJob: # prefix), used to sort and shard the jobs by WARC file; `url` is authoritative for fetching. filename: Optional[str] = None + def __str__(self) -> str: + return f'{self.url} [{self.offset},{self.length}]' + + +@dataclass(frozen=True) +class WaybackJob: + """A capture replayed from a wayback machine (`url` is its replay URL), written as one WARC record.""" + + url: str + capture: Mapping = field(compare=False) + wb: str + records_count: int = 1 + filename: Optional[str] = None # no WARC file + + def __str__(self) -> str: + return self.url + + +Job = Union[RangeJob, WaybackJob] + @dataclass(frozen=True) class RangePayload: - """Bytes output from S3 or HTTP range read.""" + """The bytes read for a job (one or more gzipped WARC records).""" - job: RangeJob + job: Job data: bytes @@ -94,3 +115,21 @@ def to_range_job(self, record: Mapping) -> RangeJob: length=int(record['length']), filename=record['filename'], ) + + +# wayback names that map to replay URLs +WAYBACK_NAMES = {'ia': 'https://web.archive.org/web'} + + +class WaybackSource: + """WaybackJobs from index records (url, timestamp, status), in the order of the records: the captures are + replayed from the wayback machine `wb` (a replay URL prefix, or a name of WAYBACK_NAMES).""" + + def __init__(self, records: Iterable[Mapping], wb: str): + from cdx_toolkit.warc import wb_replay_url + + self.wb = WAYBACK_NAMES.get(wb, wb) + self.jobs = (WaybackJob(wb_replay_url(record, self.wb), record, self.wb) for record in records) + + def iter_range_jobs(self) -> Iterator[WaybackJob]: + yield from self.jobs diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index 1e60474..5f23f7c 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -37,7 +37,7 @@ from cdx_toolkit.aws import configure_fsspec_s3 from cdx_toolkit.record_io import RecordWriter, read_records -from cdx_toolkit.repackage.jobs import RecordsSource, RepackageResult +from cdx_toolkit.repackage.jobs import RecordsSource, RepackageResult, WaybackJob, WaybackSource from cdx_toolkit.repackage.repackager import Repackager from cdx_toolkit.repackage.urls import is_s3_url, parse_s3_uri from cdx_toolkit.repackage.writers.series import generate_warc_filename @@ -79,7 +79,11 @@ def _run_worker(cfg: Dict) -> RepackageResult: if cfg.get('uvloop'): os.environ['CDXT_UVLOOP'] = '1' # the shard is in the order of the source (sorted, unless --no-sort), streamed - source = RecordsSource(read_records(cfg['shard'], 'jsonl'), warc_download_prefix=None, sort=False) + records = read_records(cfg['shard'], 'jsonl') + if cfg['wb']: + source = WaybackSource(records, cfg['wb']) + else: + source = RecordsSource(records, warc_download_prefix=None, sort=False) repackager = Repackager( source=source, prefix_path=cfg['prefix_path'], @@ -96,11 +100,18 @@ def _run_worker(cfg: Dict) -> RepackageResult: return repackager.run() +def _shard_record(job) -> dict: + """The record of a job in a shard: the WARC location with the full WARC URL as filename (read without a + download prefix), or the capture of a wayback replay.""" + if isinstance(job, WaybackJob): + return dict(job.capture) + return {'filename': job.url, 'offset': job.offset, 'length': job.length} + + def _split_source_to_shards(source, n: int, out_dir: str) -> Tuple[List[str], List[int]]: - """Drain the source once into N shard files, hashed by WARC file; the records have the full WARC URL as - filename, so they are read without a download prefix. + """Drain the source once into N shard files, hashed by WARC file (or replay URL). - Returns the shard paths and the number of range jobs in each shard.""" + Returns the shard paths and the number of jobs in each shard.""" paths = [os.path.join(out_dir, f'shard.{i}.jsonl') for i in range(n)] files = [open(p, 'w', encoding='utf-8') for p in paths] writers = [RecordWriter(f, 'jsonl') for f in files] @@ -110,7 +121,7 @@ def _split_source_to_shards(source, n: int, out_dir: str) -> Tuple[List[str], Li for job in source.iter_range_jobs(): key = (job.filename or job.url or '').encode() idx = int(hashlib.md5(key).hexdigest(), 16) % n - writers[idx].write({'filename': job.url, 'offset': job.offset, 'length': job.length}) + writers[idx].write(_shard_record(job)) counts[idx] += 1 total += 1 finally: @@ -166,6 +177,7 @@ def run_multiprocess_repackage( max_attempts=max_attempts, max_file_size=max_file_size, uvloop=uvloop, + wb=getattr(source, 'wb', None), ) for shard in shards ] diff --git a/cdx_toolkit/repackage/readers/__init__.py b/cdx_toolkit/repackage/readers/__init__.py index dccfb87..ad9c90d 100644 --- a/cdx_toolkit/repackage/readers/__init__.py +++ b/cdx_toolkit/repackage/readers/__init__.py @@ -1,43 +1,50 @@ ''' -The readers of WARC ranges, one per storage: HTTP(S) (http.py), S3 (s3.py) and Hugging Face buckets (hf.py). +The readers of jobs: WARC ranges, one reader per storage (HTTP(S) in http.py, S3 in s3.py, Hugging Face buckets +in hf.py), and captures replayed from a wayback machine (wayback.py). ''' from abc import ABC, abstractmethod import asyncio import contextlib from typing import Callable -from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.jobs import Job, WaybackJob from cdx_toolkit.repackage.urls import is_hf_url, is_s3_url class RangeReader(ABC): - """Reads the bytes of a range job from its storage.""" + """Reads the bytes of a job: a range from its storage, or a wayback replay.""" @abstractmethod - async def read(self, job: RangeJob) -> bytes: - """Exactly the bytes job.offset .. job.offset + job.length - 1 of job.url; raises (after the reader's - retries) if they cannot be read, e.g. RangeError for other bytes than requested.""" + async def read(self, job: Job) -> bytes: + """For a RangeJob exactly the bytes job.offset .. job.offset + job.length - 1 of job.url; raises (after the + reader's retries) if they cannot be read, e.g. RangeError for other bytes than requested.""" raise NotImplementedError # pragma: no cover class Readers: - """The reader of each range job, by the URL scheme of the job: S3 for s3://, the HF reader for hf://, - HTTP otherwise. The S3 and HF readers are created when they are first needed, and closed with `stack`.""" + """The reader of each job: the wayback reader for a WaybackJob; for a RangeJob by the URL scheme: S3 for + s3://, the HF reader for hf://, HTTP otherwise. The S3 and HF readers are created when they are first needed, + and closed with `stack`.""" def __init__(self, stack: contextlib.AsyncExitStack, s3_config_factory: Callable, max_attempts: int = 5, base_backoff_seconds: float = 0.5): from cdx_toolkit.repackage.readers.http import HttpRangeReader + from cdx_toolkit.repackage.readers.wayback import WaybackReader self.stack = stack self.s3_config_factory = s3_config_factory self.max_attempts = max_attempts self.base_backoff_seconds = base_backoff_seconds self._http = HttpRangeReader() + self._wayback = WaybackReader() self._s3 = None self._hf = None self._lock = asyncio.Lock() - async def for_url(self, url: str) -> RangeReader: + async def for_job(self, job: Job) -> RangeReader: + if isinstance(job, WaybackJob): + return self._wayback + url = job.url if is_s3_url(url): return await self._s3_reader() if is_hf_url(url): diff --git a/cdx_toolkit/repackage/readers/wayback.py b/cdx_toolkit/repackage/readers/wayback.py new file mode 100644 index 0000000..80cf1f1 --- /dev/null +++ b/cdx_toolkit/repackage/readers/wayback.py @@ -0,0 +1,26 @@ +''' +Captures replayed from a wayback machine (e.g. the Internet Archive), as WARC records. +''' +import logging + +from cdx_toolkit.repackage.jobs import WaybackJob +from cdx_toolkit.repackage.readers import RangeReader +from cdx_toolkit.repackage.readers.http import run_in_daemon_thread +from cdx_toolkit.repackage.warc_records import get_bytes_from_warc_record +from cdx_toolkit.warc import fetch_wb_warc + + +logger = logging.getLogger(__name__) + + +class WaybackReader(RangeReader): + """Replays a capture with fetch_wb_warc (myrequests_get: retries, and the minimum interval between requests + to the wayback machine, also across threads) and returns it as a gzipped WARC response record.""" + + async def read(self, job: WaybackJob) -> bytes: + capture = job.capture + if capture.get('mime') == 'warc/revisit': + logger.warning('revisit record being resolved for url %s %s', capture['url'], capture['timestamp']) + # in a thread, so that the event loop (other readers, the writer) is not blocked + record = await run_in_daemon_thread(fetch_wb_warc, capture, wb=job.wb) + return get_bytes_from_warc_record(record, gzip=True) diff --git a/cdx_toolkit/repackage/repackager.py b/cdx_toolkit/repackage/repackager.py index fbb7b01..069f51d 100644 --- a/cdx_toolkit/repackage/repackager.py +++ b/cdx_toolkit/repackage/repackager.py @@ -10,7 +10,7 @@ from cdx_toolkit.aws import botocore_config -from cdx_toolkit.repackage.jobs import RangeJob, RangePayload, RepackageResult +from cdx_toolkit.repackage.jobs import RangePayload, RepackageResult from cdx_toolkit.repackage.readers import Readers from cdx_toolkit.repackage.urls import is_s3_url from cdx_toolkit.repackage.writers.series import WarcFileSeries @@ -58,12 +58,13 @@ def summary(self) -> str: class Repackager: - """Copy WARC records, byte for byte, from the WARC files of the range jobs into a series of WARC files. + """Copy WARC records, byte for byte, from the WARC files of the range jobs (or the records replayed from a + wayback machine) into a series of WARC files. A pipeline of asyncio tasks: - the producer drives the (sync) source in a thread and puts its range jobs into a queue, - - N reader tasks read the bytes of the jobs, with the RangeReader for the URL scheme of each job - (HTTP, S3, Hugging Face buckets; see readers/), + - N reader tasks read the bytes of the jobs, with the RangeReader of each job + (HTTP, S3, Hugging Face buckets by the URL scheme, or the wayback reader; see readers/), - one writer task writes them to the WarcFileSeries (local files or S3 objects; see writers/). Multi-core scaling is done with processes (see multiprocess.py), each with its own pipeline. @@ -240,7 +241,7 @@ async def _run_pipeline( Args: range_jobs_queue: Queue for range jobs from the source. warc_records_queue: Queue for WARC record payloads. - readers: The readers of WARC ranges (by URL scheme). + readers: The readers of the jobs (see Readers). write_s3_client: Optional S3 client for writes S3. """ logger.info('Starting job generator, %d WARC readers, 1 WARC writer', self.num_readers) @@ -319,9 +320,9 @@ async def read_warc_records( Args: reader_id: Unique identifier for this reader task. - range_jobs_queue: Queue to read RangeJob objects from. + range_jobs_queue: Queue to read the jobs (RangeJob, WaybackJob) from. warc_records_queue: Queue to put RangePayload objects into. - readers: The readers of WARC ranges (by URL scheme). + readers: The readers of the jobs (see Readers). Returns: The reader's throughput and the number of range jobs that could not be read. @@ -334,14 +335,13 @@ async def read_warc_records( if job is _STOP: logger.info('WARC Reader %d stopping: %s', reader_id, tracker.summary()) return tracker, failed - assert isinstance(job, RangeJob) try: - reader = await readers.for_url(job.url) + reader = await readers.for_job(job) data = await reader.read(job) except Exception as e: # the capture is lost, the others are still fetched; the run fails at the end failed += 1 - logger.error('WARC Reader %d failed on %s [%d,%d]: %r', reader_id, job.url, job.offset, job.length, e) + logger.error('WARC Reader %d failed on %s: %r', reader_id, job, e) continue tracker.add(bytes_count=len(data), records_count=job.records_count) self.log_progress(f'WARC Reader {reader_id}', tracker) diff --git a/cdx_toolkit/warc.py b/cdx_toolkit/warc.py index e5e599e..5636035 100644 --- a/cdx_toolkit/warc.py +++ b/cdx_toolkit/warc.py @@ -127,6 +127,11 @@ def fake_wb_warc(url, wb_url, resp, capture): ) +def wb_replay_url(capture, wb, modifier='id_'): + """The URL of the capture in the wayback machine `wb` (with `modifier` id_: the original content).""" + return f'{wb}/{capture["timestamp"]}{modifier}/{quote(capture["url"])}' + + def fetch_wb_warc(capture, wb, modifier='id_'): for field in ('url', 'timestamp', 'status'): if field not in capture: # pragma: no cover @@ -136,9 +141,7 @@ def fetch_wb_warc(capture, wb, modifier='id_'): raise ValueError('No wayback configured') url = capture['url'] - timestamp = capture['timestamp'] - - wb_url = f'{wb}/{timestamp}{modifier}/{quote(url)}' + wb_url = wb_replay_url(capture, wb, modifier) kwargs = {} status = capture['status'] diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index 6a4c8d1..baa274c 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -179,7 +179,8 @@ the index), and can be fetched in parallel (`--parallel-readers`, `--processes`) **Internet Archive** captures have no WARC location; they are replayed from the wayback machine. Before, `--ia` did this automatically. Now pass `--wb ia` (or the URL of another wayback machine). -These are written as before, to `[-]-000000.extracted.warc.gz`, ...: +They are written like Common Crawl captures, to `[-]-001.warc.gz`, ... (before: +`-000000.extracted.warc.gz`, ...), and can be fetched with `--parallel-readers`: ```sh # old diff --git a/tests/repackage/test_readers.py b/tests/repackage/test_readers.py index 1a5e83a..d6e2f51 100644 --- a/tests/repackage/test_readers.py +++ b/tests/repackage/test_readers.py @@ -4,14 +4,16 @@ import pytest +from cdx_toolkit.repackage.jobs import RangeJob, WaybackJob from cdx_toolkit.repackage.readers import Readers from cdx_toolkit.repackage.readers.http import HttpRangeReader, RangeError, check_range_response +from cdx_toolkit.repackage.readers.wayback import WaybackReader from tests.conftest import requires_aioboto3 @requires_aioboto3 -def test_readers_by_url_scheme(): +def test_readers_by_job(): from cdx_toolkit.repackage.readers.s3 import S3RangeReader hf_reader = object() @@ -24,17 +26,25 @@ async def fake_make_hf_reader(max_attempts, base_backoff_seconds): async def main(): async with contextlib.AsyncExitStack() as stack: readers = Readers(stack, lambda: None, max_attempts=3, base_backoff_seconds=0.1) - http = await readers.for_url('https://data.commoncrawl.org/a.warc.gz') - s3 = await readers.for_url('s3://commoncrawl/a.warc.gz') - hf = await readers.for_url('hf://buckets/commoncrawl/commoncrawl/a.warc.gz') + + def for_url(url): + return readers.for_job(RangeJob(url, 0, 1)) + + http = await for_url('https://data.commoncrawl.org/a.warc.gz') + s3 = await for_url('s3://commoncrawl/a.warc.gz') + hf = await for_url('hf://buckets/commoncrawl/commoncrawl/a.warc.gz') # one reader per scheme - assert await readers.for_url('s3://commoncrawl/b.warc.gz') is s3 - assert await readers.for_url('http://example.org/b.warc.gz') is http - return http, s3, hf + assert await for_url('s3://commoncrawl/b.warc.gz') is s3 + assert await for_url('http://example.org/b.warc.gz') is http + # a wayback replay, by the type of the job (its replay URL is an http URL) + wb = 'https://web.archive.org/web' + wayback = await readers.for_job(WaybackJob(wb + '/1id_/http://example.com/', {}, wb)) + return http, s3, hf, wayback with patch('aioboto3.Session'), patch('cdx_toolkit.repackage.readers.hf.make_hf_reader', fake_make_hf_reader): - http, s3, hf = asyncio.run(main()) + http, s3, hf, wayback = asyncio.run(main()) assert isinstance(http, HttpRangeReader) + assert isinstance(wayback, WaybackReader) assert isinstance(s3, S3RangeReader) and (s3.max_attempts, s3.base_backoff_seconds) == (3, 0.1) assert hf is hf_reader diff --git a/tests/repackage/test_repackager.py b/tests/repackage/test_repackager.py index 522cee6..46e7358 100644 --- a/tests/repackage/test_repackager.py +++ b/tests/repackage/test_repackager.py @@ -22,7 +22,7 @@ from cdx_toolkit.repackage.readers.http import HttpRangeReader, RangeError from cdx_toolkit.repackage.readers.s3 import S3RangeReader from cdx_toolkit.repackage.writers.local import LocalOutputFile -from cdx_toolkit.repackage.jobs import RecordsSource +from cdx_toolkit.repackage.jobs import RecordsSource, WaybackSource from cdx_toolkit.repackage.repackager import Repackager, ThroughputTracker from tests.conftest import InlineExecutor from tests.fake_warc_server import FakeWarcServer @@ -517,6 +517,32 @@ def test_multiprocess_repackage_keeps_the_order_of_the_source(tmp_path): assert [r.rec_headers.get_header('WARC-Target-URI') for r in records] == [c['url'] for c in captures] +def test_multiprocess_repackage_of_wayback_replays(tmp_path): + """hf:// (and other non-S3 remote) output goes through the shards also for --wb: the shards hold the captures.""" + from warcio.recordbuilder import RecordBuilder + + def fake_fetch_wb_warc(capture, wb): + assert wb == 'https://web.archive.org/web' + return RecordBuilder().create_warc_record(capture['url'], 'resource', payload=io.BytesIO(b'x')) + + captures = [{'url': f'https://example.com/{i}', 'timestamp': '20240101000000', 'status': '200'} for i in range(3)] + fs = fsspec.filesystem('memory') + with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ + patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc', side_effect=fake_fetch_wb_warc): + result = multiprocess.run_multiprocess_repackage( + source=WaybackSource(captures, 'ia'), n_processes=1, readers_per_process=1, + prefix_path='memory://wb-test/RUN', writer_info={}, writer_subprefix=None, + write_paths_as_metadata_records=None, log_every_n=1, max_attempts=1, uvloop=False, + ) + try: + assert result == RepackageResult(requested=3, written=3) + with fsspec.open('memory://wb-test/RUN-001.warc.gz', 'rb') as f: + uris = [r.rec_headers.get_header('WARC-Target-URI') for r in ArchiveIterator(f) if r.rec_type == 'resource'] + assert uris == [c['url'] for c in captures] + finally: + fs.rm('/wb-test', recursive=True) + + def test_at_least_one_reader(): with pytest.raises(ValueError, match='n_parallel_readers'): Repackager(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=0) diff --git a/tests/test_cli.py b/tests/test_cli.py index f0633fa..690b12c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -363,7 +363,7 @@ def test_warc_prefix_1(tmpdir): @conditional_mock_responses def test_warc_prefix_2(tmpdir): warc_test_helper(tmpdir, 'index ia --limit 1 --url commoncrawl.org/*', '--wb ia') - assert tmpdir.listdir(fil='REPACKAGE-*.extracted.warc.gz') + assert tmpdir.listdir(fil='REPACKAGE-*.warc.gz') @pytest.mark.skipif(slow_ci(), reason='Slow CI') @@ -381,7 +381,7 @@ def test_warc_prefix_4(tmpdir): 'index https://web.archive.org/cdx/search/cdx --limit 1 --url commoncrawl.org/*', '--wb https://web.archive.org/web', ) - assert tmpdir.listdir(fil='REPACKAGE-*.extracted.warc.gz') + assert tmpdir.listdir(fil='REPACKAGE-*.warc.gz') @conditional_mock_responses diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 4ad696e..21b8c55 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -55,13 +55,13 @@ def test_repackage_with_wb_logs_revisit(tmp_path, caplog): captures = _write_captures(tmp_path / 'captures.jsonl', [dict(IA_CAPTURE, mime='warc/revisit')]) prefix = str(tmp_path / 'out') - with patch.object(cdx_toolkit.cli.repackage, 'fetch_wb_warc', return_value=_make_fake_record()) as fetch: - with caplog.at_level('WARNING', logger='cdx_toolkit.cli.repackage'): + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc', return_value=_make_fake_record()) as fetch: + with caplog.at_level('WARNING', logger='cdx_toolkit.repackage.readers.wayback'): main(['repackage', captures, '--wb', 'ia', '--prefix', prefix]) assert fetch.call_args.kwargs['wb'] == 'https://web.archive.org/web' assert 'revisit record being resolved' in caplog.text - assert list(tmp_path.glob('out-*.warc.gz')) + assert [p.name for p in tmp_path.glob('out-*')] == ['out-001.warc.gz'] def test_repackage_fetches_by_warc_location(tmp_path): @@ -90,7 +90,7 @@ def test_repackage_rejects_existing_warc_files_with_and_without_wb(tmp_path, cap captures = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE if wb else CC_CAPTURE]) wb_args = ['--wb', 'ia'] if wb else [] - with patch.object(cdx_toolkit.cli.repackage, 'fetch_wb_warc') as fetch, \ + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc') as fetch, \ patch('cdx_toolkit.repackage.readers.http.myrequests_get') as get, \ pytest.raises(SystemExit) as exc: main(['repackage', captures, '--prefix', str(tmp_path / 'out')] + wb_args) @@ -193,21 +193,39 @@ def test_repackage_with_wb_rejects_fetcher_options(tmp_path, capsys): assert '--processes cannot be used with --wb' in capsys.readouterr().err -def test_repackage_with_wb_passes_size_and_warcinfo_to_writer(tmp_path): - captures = _write_captures(tmp_path / 'captures.jsonl', []) +def test_repackage_with_wb_passes_size_and_warcinfo_to_the_repackager(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE]) - with patch.object(cdx_toolkit.warc, 'get_writer', wraps=cdx_toolkit.warc.get_writer) as get_writer: + with patch('cdx_toolkit.repackage.repackager.Repackager') as repackager: + repackager.return_value.run.return_value = RepackageResult() main([ 'repackage', captures, '--wb', 'ia', '--prefix', 'FOO', '--subprefix', 'BAR', '--size', '1MiB', - '--creator', 'me', '--description', 'test', + '--creator', 'me', '--description', 'test', '--parallel-readers', '2', ]) - prefix, subprefix, info = get_writer.call_args.args - assert (prefix, subprefix) == ('FOO', 'BAR') - assert get_writer.call_args.kwargs == {'size': 1024 * 1024} - assert info['isPartOf'] == 'FOO-BAR' - assert info['creator'] == 'me' - assert info['description'] == 'test' + kwargs = repackager.call_args.kwargs + assert (kwargs['prefix_path'], kwargs['writer_subprefix']) == ('FOO', 'BAR') + assert (kwargs['max_file_size'], kwargs['n_parallel_readers']) == (1024 * 1024, 2) + info = kwargs['writer_info'] + assert (info['isPartOf'], info['creator'], info['description']) == ('FOO-BAR', 'me', 'test') + jobs = list(kwargs['source'].iter_range_jobs()) + assert [job.url for job in jobs] == ['https://web.archive.org/web/20240101120000id_/http%3A//example.com'] + + +def test_repackage_with_wb_writes_the_warc_series(tmp_path): + """Replays are written like copied records: -001.warc.gz with the warcinfo and metadata records, also with + parallel readers.""" + captures = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE] * 3) + metadata = tmp_path / 'metadata.txt' + metadata.write_text('about this run') + + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc', side_effect=lambda *a, **k: _make_fake_record()): + main(['repackage', captures, '--wb', 'ia', '--prefix', str(tmp_path / 'out'), '--parallel-readers', '2', + '--add-metadata-record', str(metadata)]) + + assert [p.name for p in tmp_path.glob('out-*')] == ['out-001.warc.gz'] + with open(tmp_path / 'out-001.warc.gz', 'rb') as f: + assert [r.rec_type for r in ArchiveIterator(f)] == ['warcinfo', 'metadata'] + ['response'] * 3 @pytest.mark.parametrize( @@ -311,7 +329,7 @@ def test_repackage_fails_if_the_output_cannot_be_created(tmp_path): def test_repackage_with_wb_counts_failed_captures(tmp_path, capsys): path = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE, IA_CAPTURE]) fetched = [RuntimeError('404'), _make_fake_record()] - with patch.object(cdx_toolkit.cli.repackage, 'fetch_wb_warc', side_effect=fetched), \ + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc', side_effect=fetched), \ pytest.raises(SystemExit) as exc: main(['repackage', path, '--wb', 'ia', '--prefix', str(tmp_path / 'out')]) assert exc.value.code == 1 From e7c9677f546b88e2bb4c2b55e0a97a6de05970c6 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 21:47:53 +0200 Subject: [PATCH 52/91] feat(repackage): repackage() function; cdxt repackage calls it --- cdx_toolkit/cli/repackage.py | 129 +++------------ cdx_toolkit/repackage/__init__.py | 174 +++++++++++++++++++++ tests/repackage/test_repackage_function.py | 73 +++++++++ tests/unit/test_usage_errors.py | 2 +- 4 files changed, 268 insertions(+), 110 deletions(-) create mode 100644 tests/repackage/test_repackage_function.py diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index aa40cf9..4b5954c 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -1,14 +1,9 @@ from argparse import Namespace, RawDescriptionHelpFormatter import logging -import os import time -from typing import Optional -from cdx_toolkit.indexes import UsageError -from cdx_toolkit.record_io import ( - FORMAT_CHOICES, canonical_format, check_glob, format_from_path, import_pyarrow, read_records, -) -from cdx_toolkit.utils import get_version +from cdx_toolkit.record_io import FORMAT_CHOICES +from cdx_toolkit.repackage import repackage from .common import IncompleteError, add_verbose_arg, parse_size, positive_int @@ -114,43 +109,26 @@ def add_parser(subparsers): return parser -def build_warcinfo(args: Namespace, cmdline: str) -> dict: - ispartof = args.is_part_of - if not ispartof: - ispartof = args.prefix - if args.subprefix: - ispartof += '-' + args.subprefix - - info = { - 'software': 'pypi_cdx_toolkit/' + get_version(), - 'isPartOf': ispartof, - 'description': args.description or 'warc extraction generated with: ' + cmdline, - 'format': 'WARC file version 1.0', # todo: if we directly read a warc, have this match the warc - } - if args.creator: - info['creator'] = args.creator - if args.operator: - info['operator'] = args.operator - return info - - def run(args: Namespace, cmdline: str): - index_format = None if args.index_format == 'auto' else canonical_format(args.index_format) - try: - check_glob(args.input) - except ValueError as e: - raise UsageError(str(e)) from None - if index_format == 'parquet' or (index_format is None and format_from_path(args.input) == 'parquet'): - try: - import_pyarrow() - except RuntimeError as e: - raise UsageError(str(e)) from None - check_output_is_new(args.prefix, args.subprefix) - records = read_records(args.input, index_format) - info = build_warcinfo(args, cmdline) start_time = time.time() - - result = repackage_captures(args, records, info) + result = repackage( + args.input, + args.prefix, + subprefix=args.subprefix, + size=args.size, + warc_download_prefix=args.warc_download_prefix, + wb=args.wb, + processes=args.processes, + parallel_readers=args.parallel_readers, + metadata_records=args.add_metadata_record, + sort=not args.no_sort, + log_every_n=args.log_every_n, + creator=args.creator, + operator=args.operator, + is_part_of=args.is_part_of, + description=args.description or 'warc extraction generated with: ' + cmdline, + index_format=args.index_format, + ) LOGGER.info('wrote %d of %d captures to WARC files (%d failed) in %.1f seconds', result.written, result.requested, result.failed, time.time() - start_time) @@ -159,70 +137,3 @@ def run(args: Namespace, cmdline: str): f'{result.failed} of {result.requested} captures could not be fetched (see the log), ' f'{result.written} were written' ) - - -def check_output_is_new(prefix: str, subprefix: Optional[str]) -> None: - """Every run numbers its files from the start: never overwrite (or mix with) the files of an earlier run, with - or without --wb.""" - from cdx_toolkit.repackage.writers.series import existing_warc_files - - existing = existing_warc_files(prefix, subprefix) - if existing: - raise UsageError( - f'{len(existing)} WARC file(s) with this --prefix/--subprefix exist already, e.g. {existing[0]}; ' - 'use another prefix or subprefix, or remove them' - ) - - -def repackage_captures(args: Namespace, records, info: dict): - """Copy the WARC records byte-for-byte, or replay them from a wayback machine with --wb, with the async - repackager (optionally multi-process).""" - import fsspec - - from cdx_toolkit.repackage.urls import is_hf_url - from cdx_toolkit.repackage.jobs import MissingWarcLocation, RecordsSource, WaybackSource - try: - if args.wb: - # the minimum interval between requests to the wayback machine only holds within a process - if args.processes > 1: - raise UsageError('--processes cannot be used with --wb') - source = WaybackSource(records, args.wb) - else: - source = RecordsSource(records, args.warc_download_prefix, sort=not args.no_sort) - - # make sure the output directory exists - prefix_fs, prefix_fs_path = fsspec.url_to_fs(args.prefix) - prefix_fs.makedirs(prefix_fs._parent(prefix_fs_path), exist_ok=True) - - # The WARC writer supports local and S3 output; hf:// output is staged locally by the multi-process mode. - if args.processes > 1 or is_hf_url(args.prefix): - from cdx_toolkit.repackage.multiprocess import run_multiprocess_repackage - - return run_multiprocess_repackage( - source=source, - n_processes=args.processes, - readers_per_process=args.parallel_readers, - prefix_path=args.prefix, - writer_info=info, - writer_subprefix=args.subprefix, - write_paths_as_metadata_records=args.add_metadata_record, - log_every_n=args.log_every_n, - max_attempts=5, - uvloop=os.environ.get('CDXT_UVLOOP') == '1', - max_file_size=args.size, - ) - - from cdx_toolkit.repackage.repackager import Repackager - - return Repackager( - source=source, - prefix_path=args.prefix, - writer_info=info, - writer_subprefix=args.subprefix, - write_paths_as_metadata_records=args.add_metadata_record, - log_every_n=args.log_every_n, - n_parallel_readers=args.parallel_readers, - max_file_size=args.size, - ).run() - except MissingWarcLocation as e: - raise UsageError(str(e)) diff --git a/cdx_toolkit/repackage/__init__.py b/cdx_toolkit/repackage/__init__.py index e69de29..a2a8de9 100644 --- a/cdx_toolkit/repackage/__init__.py +++ b/cdx_toolkit/repackage/__init__.py @@ -0,0 +1,174 @@ +''' +Write captures to WARC files: copy their WARC records byte-for-byte, or replay them from a wayback machine. + +The Python equivalent of `cdxt repackage`: + + from cdx_toolkit.indexes import IndexQuery, open_index + from cdx_toolkit.repackage import repackage + + captures = open_index('cc').records(IndexQuery(urls='commoncrawl.org/*', limit=10)) + result = repackage(captures, prefix='out/COMMONCRAWL') +''' +import logging +import os +from typing import Iterable, Mapping, Optional, Sequence, Union + +from cdx_toolkit.indexes import UsageError +from cdx_toolkit.repackage.jobs import RepackageResult + + +logger = logging.getLogger(__name__) + + +def repackage( + captures: Union[str, Iterable[Mapping]], + prefix: str = 'REPACKAGE', + *, + subprefix: Optional[str] = None, + size: int = 1_000_000_000, + warc_download_prefix: str = 'https://data.commoncrawl.org', + wb: Optional[str] = None, + processes: int = 1, + parallel_readers: int = 1, + metadata_records: Sequence[str] = (), + sort: bool = True, + log_every_n: int = 1000, + creator: Optional[str] = None, + operator: Optional[str] = None, + is_part_of: Optional[str] = None, + description: Optional[str] = None, + index_format: Optional[str] = None, +) -> RepackageResult: + """Write the captures to the WARC files [-]-001.warc.gz, -002, ... (local, s3:// or hf://). + + Args: + captures: index records (e.g. IndexBackend.records()), or the path of an index file as written by + `cdxt index` (local or remote, may be a glob; '-' = stdin). + prefix, subprefix: the output files; files of this prefix and subprefix must not exist yet. + size: target byte size of each WARC file. + warc_download_prefix: where the WARC files of the captures (filename, offset, length) are read from: + https://, s3:// or hf://buckets/... + wb: replay the captures from this wayback machine instead ('ia' = the Internet Archive); for captures + without a WARC location. + processes, parallel_readers: worker processes, and concurrent readers per process. + metadata_records: files (local or remote) added as a metadata record to each WARC file. + sort: sort the captures by WARC file and offset (better read locality, keeps the captures in memory). + log_every_n: log the progress every N records (0 = off). + creator, operator, is_part_of, description: fields of the warcinfo record. + index_format: format of the index file, if `captures` is a path (default: detected). + + Returns the number of captures requested, written, and failed (could not be fetched; the others are + written). Raises UsageError for invalid options, and if the output cannot be written.""" + if wb and processes > 1: + # the minimum interval between requests to the wayback machine only holds within a process + raise UsageError('--processes cannot be used with --wb') + if isinstance(captures, str): + captures = read_index_file(captures, index_format) + check_output_is_new(prefix, subprefix) + info = warcinfo(prefix, subprefix, creator, operator, is_part_of, description) + return _run(captures, prefix, subprefix, size, warc_download_prefix, wb, processes, parallel_readers, + list(metadata_records), sort, log_every_n, info) + + +def read_index_file(path: str, index_format: Optional[str] = None) -> Iterable[dict]: + from cdx_toolkit.record_io import canonical_format, check_glob, format_from_path, import_pyarrow, read_records + + index_format = None if index_format in (None, 'auto') else canonical_format(index_format) + try: + check_glob(path) + except ValueError as e: + raise UsageError(str(e)) from None + if index_format == 'parquet' or (index_format is None and format_from_path(path) == 'parquet'): + try: + import_pyarrow() + except RuntimeError as e: + raise UsageError(str(e)) from None + return read_records(path, index_format) + + +def check_output_is_new(prefix: str, subprefix: Optional[str]) -> None: + """Every run numbers its files from the start: never overwrite (or mix with) the files of an earlier run.""" + from cdx_toolkit.repackage.writers.series import existing_warc_files + + existing = existing_warc_files(prefix, subprefix) + if existing: + raise UsageError( + f'{len(existing)} WARC file(s) with this --prefix/--subprefix exist already, e.g. {existing[0]}; ' + 'use another prefix or subprefix, or remove them' + ) + + +def warcinfo(prefix: str, subprefix: Optional[str] = None, creator: Optional[str] = None, + operator: Optional[str] = None, is_part_of: Optional[str] = None, + description: Optional[str] = None) -> dict: + """The fields of the warcinfo record at the start of each WARC file.""" + from cdx_toolkit.utils import get_version + + if not is_part_of: + is_part_of = prefix + ('-' + subprefix if subprefix else '') + info = { + 'software': 'pypi_cdx_toolkit/' + get_version(), + 'isPartOf': is_part_of, + 'description': description or 'warc extraction generated with cdx_toolkit', + 'format': 'WARC file version 1.0', # todo: if we directly read a warc, have this match the warc + } + if creator: + info['creator'] = creator + if operator: + info['operator'] = operator + return info + + +def _run(captures, prefix, subprefix, size, warc_download_prefix, wb, processes, parallel_readers, + metadata_records, sort, log_every_n, info) -> RepackageResult: + """Copy the WARC records byte-for-byte, or replay them from a wayback machine, with the async repackager + (optionally multi-process).""" + import fsspec + + from cdx_toolkit.repackage.jobs import MissingWarcLocation, RecordsSource, WaybackSource + from cdx_toolkit.repackage.urls import is_hf_url + try: + if wb: + source = WaybackSource(captures, wb) + else: + source = RecordsSource(captures, warc_download_prefix, sort=sort) + except MissingWarcLocation as e: + raise UsageError(str(e)) from None + + # make sure the output directory exists + prefix_fs, prefix_fs_path = fsspec.url_to_fs(prefix) + prefix_fs.makedirs(prefix_fs._parent(prefix_fs_path), exist_ok=True) + + try: + # The WARC writer supports local and S3 output; hf:// output is staged locally by the multi-process mode. + if processes > 1 or is_hf_url(prefix): + from cdx_toolkit.repackage.multiprocess import run_multiprocess_repackage + + return run_multiprocess_repackage( + source=source, + n_processes=processes, + readers_per_process=parallel_readers, + prefix_path=prefix, + writer_info=info, + writer_subprefix=subprefix, + write_paths_as_metadata_records=metadata_records, + log_every_n=log_every_n, + max_attempts=5, + uvloop=os.environ.get('CDXT_UVLOOP') == '1', + max_file_size=size, + ) + + from cdx_toolkit.repackage.repackager import Repackager + + return Repackager( + source=source, + prefix_path=prefix, + writer_info=info, + writer_subprefix=subprefix, + write_paths_as_metadata_records=metadata_records, + log_every_n=log_every_n, + n_parallel_readers=parallel_readers, + max_file_size=size, + ).run() + except MissingWarcLocation as e: # with sort=False, the records are read while the jobs are run + raise UsageError(str(e)) from None diff --git a/tests/repackage/test_repackage_function.py b/tests/repackage/test_repackage_function.py new file mode 100644 index 0000000..878087d --- /dev/null +++ b/tests/repackage/test_repackage_function.py @@ -0,0 +1,73 @@ +"""repackage(), the Python equivalent of `cdxt repackage`, with open_index() and IndexQuery for `cdxt index`.""" +import json +from unittest.mock import patch + +import pytest +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index +from cdx_toolkit.repackage import repackage +from cdx_toolkit.repackage.jobs import RepackageResult +from tests.fake_warc_server import FakeWarcServer + + +SERVER = FakeWarcServer('https://data.example.org') +CAPTURES = SERVER.add_captures('crawl-data/0.warc.gz', [f'https://example.com/{i}' for i in range(3)]) + + +def warc_records(path): + with open(path, 'rb') as f: + return [(r.rec_type, r.rec_headers.get_header('WARC-Target-URI')) for r in ArchiveIterator(f)] + + +def fetch_from_server(): + return patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=SERVER.get) + + +def test_repackage_records(tmp_path): + with fetch_from_server(): + result = repackage(CAPTURES, str(tmp_path / 'out'), warc_download_prefix='https://data.example.org') + + assert result == RepackageResult(requested=3, written=3) + assert warc_records(tmp_path / 'out-001.warc.gz') == [('warcinfo', None)] + [ + ('resource', c['url']) for c in CAPTURES] + + +def test_repackage_index_file(tmp_path): + path = tmp_path / 'captures.jsonl' + path.write_text(''.join(json.dumps(c) + '\n' for c in CAPTURES)) + with fetch_from_server(): + result = repackage(str(path), str(tmp_path / 'out'), warc_download_prefix='https://data.example.org', + subprefix='SUB', creator='me') + + assert result.written == 3 + with open(tmp_path / 'out-SUB-001.warc.gz', 'rb') as f: + warcinfo = next(iter(ArchiveIterator(f))) + fields = warcinfo.content_stream().read().decode() + assert 'isPartOf: ' + str(tmp_path / 'out') + '-SUB' in fields + assert 'creator: me' in fields + assert 'description: warc extraction generated with cdx_toolkit' in fields + + +def test_open_index_and_repackage(tmp_path): + """The two steps in Python, as `cdxt index INDEX --domain example.com | cdxt repackage`.""" + index = tmp_path / 'index.cdx' + index.write_text(''.join( + f'com,example)/{c["url"].rsplit("/", 1)[1]} 20240101000000 {json.dumps(c)}\n' for c in CAPTURES)) + + captures = open_index(str(index)).records(IndexQuery(domains='example.com', limit=2)) + with fetch_from_server(): + result = repackage(captures, str(tmp_path / 'out'), warc_download_prefix='https://data.example.org') + + assert result == RepackageResult(requested=2, written=2) + assert [url for _, url in warc_records(tmp_path / 'out-001.warc.gz')[1:]] == [c['url'] for c in CAPTURES[:2]] + + +def test_repackage_usage_errors(tmp_path): + (tmp_path / 'out-001.warc.gz').write_bytes(b'earlier run') + with pytest.raises(UsageError, match='exist already'): + repackage(CAPTURES, str(tmp_path / 'out')) + with pytest.raises(UsageError, match='--processes cannot be used with --wb'): + repackage(CAPTURES, str(tmp_path / 'new'), wb='ia', processes=2) + with pytest.raises(UsageError, match='no WARC location'): + repackage([{'url': 'https://example.com/'}], str(tmp_path / 'new')) diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py index 63a7bcf..14980f4 100644 --- a/tests/unit/test_usage_errors.py +++ b/tests/unit/test_usage_errors.py @@ -49,7 +49,7 @@ def test_usage_errors(capsys, args, error): ) def test_parquet_without_pyarrow(capsys, command): with patch('cdx_toolkit.cli.index.import_pyarrow', side_effect=RuntimeError('needs pyarrow')), \ - patch('cdx_toolkit.cli.repackage.import_pyarrow', side_effect=RuntimeError('needs pyarrow')): + patch('cdx_toolkit.record_io.import_pyarrow', side_effect=RuntimeError('needs pyarrow')): with pytest.raises(SystemExit): main(command) assert 'needs pyarrow' in capsys.readouterr().err From 03ed0618314b11b4891cc86da5c6946a8a6ab3fb Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 21:53:57 +0200 Subject: [PATCH 53/91] deprecate CDXToolkitWARCWriter; document the Python API of index and repackage --- CHANGELOG.md | 3 ++ README.md | 34 ++++++++++++++++++-- cdx_toolkit/warc.py | 9 +++++- docs/cli-migration-guide.md | 17 ++++++++-- examples/iter-and-warc.py | 49 ++++++++++++----------------- tests/repackage/test_warc_writer.py | 6 ++-- tests/unit/test_warc.py | 8 +++++ 7 files changed, 89 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29cada7..253d7c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ + S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual addressing, no payload signing, 2 concurrent requests) unless configured by the user; `CDXT_AWS_DEFAULTS=0` turns them off; also for Athena and, as far as DuckDB has equivalents, for DuckDB (fixes DuckDB's `http_timeout`, which is in seconds) + + Python API of the new commands: `cdx_toolkit.indexes.open_index()` with `IndexQuery` (`records()`, `count()`, `explain()`) + and `cdx_toolkit.repackage.repackage()`; `cdxt index` and `cdxt repackage` run the same code + + deprecated `cdx_toolkit.warc.get_writer()` and `CDXToolkitWARCWriter` (FutureWarning): use `repackage()` + new extras: `athena`, `duckdb`, `hf`, `parquet`; `fsspec` and `aiofiles` are now required - 0.9.38 diff --git a/README.md b/README.md index a4ec8ec..f5e95aa 100644 --- a/README.md +++ b/README.md @@ -304,7 +304,34 @@ for full details. Add -v (or -vv) to see what's going on under the hood. ## Python programming example Everything that you can do on the command line, and much more, can -be done by writing a Python program. +be done by writing a Python program. `cdxt index` and `cdxt repackage` +run the same code as `open_index()` with an `IndexQuery`, and `repackage()`: + +``` +from cdx_toolkit.indexes import IndexQuery, open_index +from cdx_toolkit.repackage import repackage + +index = open_index('cc') # any INDEX of `cdxt index`, e.g. 'cc-url-index-s3' or 'athena://...' +query = IndexQuery(urls='commoncrawl.org/*', crawl='CC-MAIN-2024-33', limit=10) + +print(index.count(query)) # like --count +print(index.explain(query)) # like --dry-run +for capture in index.records(query): + print(capture['url'], capture['timestamp']) + +result = repackage(index.records(query), prefix='out/COMMONCRAWL', size=100_000_000) +print(result.written, 'written,', result.failed, 'failed') +``` + +The fields of `IndexQuery` are the options of `cdxt index` (`urls`, `hostnames`, `domains`, +`url_file`, `crawl`, `from_ts`, `to`, `filters`, `languages`, `url_fgrep`, `limit`, `sql`, ...), and the +arguments of `repackage()` those of `cdxt repackage` (`subprefix`, `size`, `warc_download_prefix`, +`wb`, `processes`, `parallel_readers`, `metadata_records`, ...). `repackage()` also takes the path of +an index file. To write the captures to a file like `cdxt index -o`, use +`cdx_toolkit.record_io.write_records(index.records(query), 'captures.csv', 'csv', fixed_schema=index.fixed_schema)`. + +`CDXFetcher` is the lower-level client of the CDX API, which can also +fetch the content of single captures: ``` import cdx_toolkit @@ -331,8 +358,9 @@ You can also fetch the content of the web capture as bytes: print(obj.content) ``` -There's a full example of iterating and selecting a subset of captures -to write into an extracted WARC file in [examples/iter-and-warc.py](examples/iter-and-warc.py) +There's a full example of selecting a subset of captures and writing them +into a WARC file in [examples/iter-and-warc.py](examples/iter-and-warc.py). +(`cdx_toolkit.warc.get_writer()` is deprecated: use `repackage()`.) ## Filter syntax diff --git a/cdx_toolkit/warc.py b/cdx_toolkit/warc.py index 5636035..c8a79db 100644 --- a/cdx_toolkit/warc.py +++ b/cdx_toolkit/warc.py @@ -3,6 +3,7 @@ import datetime import logging import os +import warnings try: import fsspec @@ -199,11 +200,17 @@ def fetch_warc_record(capture, warc_download_prefix): class CDXToolkitWARCWriter: - """Writer for WARC files. + """Writer for WARC files (deprecated: use cdx_toolkit.repackage.repackage()). The fsspec package is used for writting to local or remote file system, e.g., S3.""" def __init__(self, prefix, subprefix, info, size=1000000000, gzip=True, warc_version=None): + warnings.warn( + 'cdx_toolkit.warc.get_writer() and CDXToolkitWARCWriter are deprecated and will be removed in ' + 'cdx_toolkit 1.0; use cdx_toolkit.repackage.repackage()', + FutureWarning, + stacklevel=2, + ) self.prefix = prefix self.subprefix = subprefix self.info = info diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index baa274c..35a4354 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -300,5 +300,18 @@ The `repackage` command of the unreleased `feat/warc-range-sources` branch (#73) ## Python API -The Python API (`cdx_toolkit.CDXFetcher`, `cdx.iter()`, `cdx_toolkit.warc.get_writer()`, ...) has -not changed. Only the command line tool is affected. +`cdx_toolkit.CDXFetcher`, `cdx.iter()`, `cdx.get_size_estimate()` and `capture.content` have not +changed. New: the Python equivalents of the new commands, which run the same code as the command line +(see the README): + +```python +from cdx_toolkit.indexes import IndexQuery, open_index +from cdx_toolkit.repackage import repackage + +index = open_index('cc') # cdxt index cc +query = IndexQuery(urls='commoncrawl.org/*', limit=10) # --url 'commoncrawl.org/*' --limit 10 +repackage(index.records(query), prefix='COMMONCRAWL') # | cdxt repackage --prefix COMMONCRAWL +``` + +`cdx_toolkit.warc.get_writer()` (`CDXToolkitWARCWriter`) is deprecated and will be removed in +cdx_toolkit 1.0; `repackage()` replaces it. diff --git a/examples/iter-and-warc.py b/examples/iter-and-warc.py index 1d1b820..75e66fc 100755 --- a/examples/iter-and-warc.py +++ b/examples/iter-and-warc.py @@ -1,33 +1,24 @@ #!/usr/bin/env python +''' +Select captures from Common Crawl's CDX index and write them into a WARC file, +like `cdxt index cc --url 'commoncrawl.org/blog/*' --filter =status:200 --limit 10 | cdxt repackage ...` +''' +from cdx_toolkit.indexes import IndexQuery, open_index +from cdx_toolkit.repackage import repackage -import cdx_toolkit +index = open_index('cc') +query = IndexQuery(urls='commoncrawl.org/blog/*', filters='=status:200', limit=10) -cdx = cdx_toolkit.CDXFetcher(source='cc') -url = 'commoncrawl.org/blog/*' +captures = [] +for capture in index.records(query): + print('extracting url', capture['url'], 'timestamp', capture['timestamp']) + captures.append(capture) -warcinfo = { - 'software': 'pypi_cdx_toolkit iter-and-warc example', - 'isPartOf': 'EXAMPLE-COMMONCRAWL', - 'description': 'warc extraction', - 'format': 'WARC file version 1.0', -} - -with cdx_toolkit.warc.get_writer('EXAMPLE', 'COMMONCRAWL', warcinfo, warc_version='1.1') as writer: - for obj in cdx.iter(url, limit=10): - url = obj['url'] - status = obj['status'] - timestamp = obj['timestamp'] - - print('considering extracting url', url, 'timestamp', timestamp) - if status != '200': - print(' skipping because status was {}, not 200'.format(status)) - continue - - try: - record = obj.fetch_warc_record() - except RuntimeError: - print(' skipping capture for RuntimeError 404: %s %s', url, timestamp) - continue - writer.write_record(record) - - print(' wrote', url) +result = repackage( + captures, + prefix='EXAMPLE', + subprefix='COMMONCRAWL', + is_part_of='EXAMPLE-COMMONCRAWL', + description='warc extraction', +) +print('wrote', result.written, 'captures to EXAMPLE-COMMONCRAWL-001.warc.gz, ...;', result.failed, 'failed') diff --git a/tests/repackage/test_warc_writer.py b/tests/repackage/test_warc_writer.py index 910411c..cecea40 100644 --- a/tests/repackage/test_warc_writer.py +++ b/tests/repackage/test_warc_writer.py @@ -31,7 +31,8 @@ def test_write_to_local(prefix, gzip, tmpdir): full_prefix = str(tmpdir) + '/' + prefix fs, fs_prefix_path = fsspec.url_to_fs(full_prefix) - writer = cdx_toolkit.warc.get_writer(full_prefix, None, info, gzip=gzip) + with pytest.warns(FutureWarning, match='deprecated'): + writer = cdx_toolkit.warc.get_writer(full_prefix, None, info, gzip=gzip) # single record input_resource_record_text = 'foo bar text' @@ -82,7 +83,8 @@ def test_write_to_s3(s3_tmpdir): fs, fs_prefix_path = fsspec.url_to_fs(s3_tmpdir) - writer = cdx_toolkit.warc.get_writer(s3_tmpdir, None, info) + with pytest.warns(FutureWarning, match='deprecated'): + writer = cdx_toolkit.warc.get_writer(s3_tmpdir, None, info) # single record input_resource_record_text = 'foo bar text' diff --git a/tests/unit/test_warc.py b/tests/unit/test_warc.py index 2991f16..7371eae 100644 --- a/tests/unit/test_warc.py +++ b/tests/unit/test_warc.py @@ -3,6 +3,14 @@ from unittest.mock import Mock import pytest +# the tests of the deprecated writer (see test_writer_is_deprecated) +pytestmark = pytest.mark.filterwarnings('ignore:cdx_toolkit.warc.get_writer:FutureWarning') + + +def test_writer_is_deprecated(tmp_path): + with pytest.warns(FutureWarning, match='use cdx_toolkit.repackage.repackage'): + cdx_toolkit.warc.get_writer(str(tmp_path / 'out'), None, {}) + def test_wb_redir_to_original(): location = 'https://web.archive.org/web/20110209062054id_/http://commoncrawl.org/' From f21cad69ea74e7146c03aa53c771260fad6e93e5 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 23:08:40 +0200 Subject: [PATCH 54/91] fix(repackage): --wb checks the captures: url, timestamp and status are needed, Common Crawl captures are refused cdxt index cc | cdxt repackage --wb ia replayed Common Crawl captures from the Internet Archive (another archive's captures); the old cdxt --cc warc refused --wb. --- CHANGELOG.md | 2 ++ README.md | 4 ++- cdx_toolkit/repackage/__init__.py | 6 ++-- cdx_toolkit/repackage/jobs.py | 42 ++++++++++++++++++++++++-- tests/repackage/test_jobs.py | 49 ++++++++++++++++++++++++++++++- tests/unit/test_cli.py | 13 ++++++++ 6 files changed, 108 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 253d7c1..ad21514 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ + S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual addressing, no payload signing, 2 concurrent requests) unless configured by the user; `CDXT_AWS_DEFAULTS=0` turns them off; also for Athena and, as far as DuckDB has equivalents, for DuckDB (fixes DuckDB's `http_timeout`, which is in seconds) + + `cdxt repackage --wb` checks the captures before replaying them: `url` (http/https), `timestamp` and `status` + are needed, and Common Crawl captures are refused (as `cdxt --cc warc` refused `--wb`) + Python API of the new commands: `cdx_toolkit.indexes.open_index()` with `IndexQuery` (`records()`, `count()`, `explain()`) and `cdx_toolkit.repackage.repackage()`; `cdxt index` and `cdxt repackage` run the same code + deprecated `cdx_toolkit.warc.get_writer()` and `CDXToolkitWARCWriter` (FutureWarning): use `repackage()` diff --git a/README.md b/README.md index f5e95aa..d32ed9a 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,9 @@ replayed from the wayback machine given with `--wb` (`ia` is a shortcut for `https://web.archive.org/web`) and written to the same `[-]-001.warc.gz`, ... series, also with `--parallel-readers` and `--add-metadata-record` (`--processes` cannot be used with `--wb`: the requests to the wayback machine -are rate-limited per process). +are rate-limited per process). A replay needs the `url` (http or https), `timestamp` and `status` of +each capture. Common Crawl captures (`filename` `crawl-data/...`) are refused with `--wb`: they are +copied from their WARC files, while a replay would get another archive's capture. In both cases, if WARC files with the same prefix and subprefix exist already (e.g. from an earlier run), `cdxt repackage` stops before fetching (exit diff --git a/cdx_toolkit/repackage/__init__.py b/cdx_toolkit/repackage/__init__.py index a2a8de9..af89962 100644 --- a/cdx_toolkit/repackage/__init__.py +++ b/cdx_toolkit/repackage/__init__.py @@ -125,14 +125,14 @@ def _run(captures, prefix, subprefix, size, warc_download_prefix, wb, processes, (optionally multi-process).""" import fsspec - from cdx_toolkit.repackage.jobs import MissingWarcLocation, RecordsSource, WaybackSource + from cdx_toolkit.repackage.jobs import InvalidWaybackCapture, MissingWarcLocation, RecordsSource, WaybackSource from cdx_toolkit.repackage.urls import is_hf_url try: if wb: source = WaybackSource(captures, wb) else: source = RecordsSource(captures, warc_download_prefix, sort=sort) - except MissingWarcLocation as e: + except (MissingWarcLocation, InvalidWaybackCapture) as e: raise UsageError(str(e)) from None # make sure the output directory exists @@ -170,5 +170,5 @@ def _run(captures, prefix, subprefix, size, warc_download_prefix, wb, processes, n_parallel_readers=parallel_readers, max_file_size=size, ).run() - except MissingWarcLocation as e: # with sort=False, the records are read while the jobs are run + except (MissingWarcLocation, InvalidWaybackCapture) as e: # streamed sources: read while the jobs are run raise UsageError(str(e)) from None diff --git a/cdx_toolkit/repackage/jobs.py b/cdx_toolkit/repackage/jobs.py index 3376bc1..b1f9aaa 100644 --- a/cdx_toolkit/repackage/jobs.py +++ b/cdx_toolkit/repackage/jobs.py @@ -3,7 +3,9 @@ replay), and the result of a run. ''' import logging +import re from dataclasses import dataclass, field +from urllib.parse import urlparse from typing import Iterable, Iterator, Mapping, Optional, Union @@ -121,15 +123,49 @@ def to_range_job(self, record: Mapping) -> RangeJob: WAYBACK_NAMES = {'ia': 'https://web.archive.org/web'} +WAYBACK_FIELDS = ('url', 'timestamp', 'status') + +# the WARC files of Common Crawl, e.g. crawl-data/CC-MAIN-2024-33/segments/..., also as a full URL +COMMON_CRAWL_WARC = re.compile(r'(^|/)crawl-data/') + + +class InvalidWaybackCapture(ValueError): + """A record cannot be replayed from a wayback machine: it lacks a field, or is a Common Crawl capture.""" + + class WaybackSource: """WaybackJobs from index records (url, timestamp, status), in the order of the records: the captures are replayed from the wayback machine `wb` (a replay URL prefix, or a name of WAYBACK_NAMES).""" def __init__(self, records: Iterable[Mapping], wb: str): - from cdx_toolkit.warc import wb_replay_url - self.wb = WAYBACK_NAMES.get(wb, wb) - self.jobs = (WaybackJob(wb_replay_url(record, self.wb), record, self.wb) for record in records) + self.jobs = (self.to_job(record) for record in records) def iter_range_jobs(self) -> Iterator[WaybackJob]: yield from self.jobs + + def to_job(self, record: Mapping) -> WaybackJob: + from cdx_toolkit.warc import wb_replay_url + + if COMMON_CRAWL_WARC.search(str(record.get('filename') or '')): + # the old `cdxt --cc warc` refused --wb as well + raise InvalidWaybackCapture( + f'capture {record.get("url")} is a Common Crawl capture (filename {record["filename"]}): it is ' + 'copied from its WARC file, do not use --wb' + ) + missing = [name for name in WAYBACK_FIELDS if record.get(name) in (None, '')] + if missing: + raise InvalidWaybackCapture( + f'capture {record.get("url")} has no {", ".join(missing)}, which --wb needs to replay it ' + f'({", ".join(WAYBACK_FIELDS)})' + ) + url, timestamp = str(record['url']), str(record['timestamp']) + parsed = urlparse(url) + if parsed.scheme not in ('http', 'https') or not parsed.netloc: + raise InvalidWaybackCapture(f'capture url {url!r} is not an http(s) URL, which --wb needs to replay it') + if not (timestamp.isdigit() and 4 <= len(timestamp) <= 14): + raise InvalidWaybackCapture(f'capture {url} has an invalid timestamp {timestamp!r} for --wb, ' + 'e.g. 20080328121314') + # the replay compares the status as a string (Parquet and JSON input can have numbers) + capture = {**record, 'url': url, 'timestamp': timestamp, 'status': str(record['status'])} + return WaybackJob(wb_replay_url(capture, self.wb), capture, self.wb) diff --git a/tests/repackage/test_jobs.py b/tests/repackage/test_jobs.py index 02a0e5d..b7d3673 100644 --- a/tests/repackage/test_jobs.py +++ b/tests/repackage/test_jobs.py @@ -1,4 +1,8 @@ -from cdx_toolkit.repackage.jobs import join_warc_url +import pytest + +from cdx_toolkit.repackage.jobs import InvalidWaybackCapture, WaybackSource, join_warc_url + +IA_CAPTURE = {'url': 'http://example.com/a b', 'timestamp': '20240101120000', 'status': '200'} def test_join_warc_url_relative(): @@ -24,3 +28,46 @@ def test_join_warc_url_empty_prefix(): def test_join_warc_url_absolute_filename_passthrough(): assert join_warc_url('https://data.commoncrawl.org', 's3://cc/x.warc.gz') == 's3://cc/x.warc.gz' assert join_warc_url('', 'https://host/x.warc.gz') == 'https://host/x.warc.gz' + + +def wayback_job(record, wb='ia'): + return next(WaybackSource([record], wb).iter_range_jobs()) + + +def test_wayback_job(): + job = wayback_job(dict(IA_CAPTURE, status=200, mime='text/html')) + assert job.url == 'https://web.archive.org/web/20240101120000id_/http%3A//example.com/a%20b' + assert job.wb == 'https://web.archive.org/web' + assert job.capture == {**IA_CAPTURE, 'mime': 'text/html'} # the status as a string, like the CDX API's + assert wayback_job(IA_CAPTURE, 'https://wb.example.org/web').url.startswith('https://wb.example.org/web/') + + +def test_wayback_jobs_from_another_wayback_with_warc_location(): + """pywb servers return filename/offset/length too, relative to their own storage: replayed, not copied.""" + capture = dict(IA_CAPTURE, filename='collections/my/archive/x.warc.gz', offset='0', length='10') + assert wayback_job(capture, 'https://wb.example.org/web').capture == capture + + +@pytest.mark.parametrize('filename', [ + 'crawl-data/CC-MAIN-2024-33/segments/1/warc/x.warc.gz', + 'https://data.commoncrawl.org/crawl-data/CC-NEWS/2024/01/x.warc.gz', + 's3://commoncrawl/crawl-data/CC-MAIN-2024-33/x.warc.gz', +]) +def test_wayback_rejects_common_crawl_captures(filename): + with pytest.raises(InvalidWaybackCapture, match='Common Crawl capture .* do not use --wb'): + wayback_job(dict(IA_CAPTURE, filename=filename, offset='0', length='10')) + + +@pytest.mark.parametrize('change, error', [ + ({'url': None}, 'has no url'), + ({'timestamp': ''}, 'has no timestamp'), + ({'status': None}, 'has no status'), + ({'url': 'example.com/a'}, 'not an http.s. URL'), + ({'url': 'ftp://example.com/a'}, 'not an http.s. URL'), + ({'url': 'http://'}, 'not an http.s. URL'), + ({'timestamp': '2024-01-01'}, 'invalid timestamp'), + ({'timestamp': '202'}, 'invalid timestamp'), +]) +def test_wayback_rejects_invalid_captures(change, error): + with pytest.raises(InvalidWaybackCapture, match=error): + wayback_job({**IA_CAPTURE, **change}) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 21b8c55..c34832f 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -212,6 +212,19 @@ def test_repackage_with_wb_passes_size_and_warcinfo_to_the_repackager(tmp_path): assert [job.url for job in jobs] == ['https://web.archive.org/web/20240101120000id_/http%3A//example.com'] +def test_repackage_with_wb_rejects_common_crawl_captures(tmp_path, capsys): + """cdxt index cc ... | cdxt repackage --wb ia would replay the captures from another archive.""" + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc') as fetch, pytest.raises(SystemExit) as exc: + main(['repackage', captures, '--wb', 'ia', '--prefix', str(tmp_path / 'out')]) + + assert exc.value.code == 2 + assert 'is a Common Crawl capture' in capsys.readouterr().err + fetch.assert_not_called() + assert list(tmp_path.glob('out-*')) == [] + + def test_repackage_with_wb_writes_the_warc_series(tmp_path): """Replays are written like copied records: -001.warc.gz with the warcinfo and metadata records, also with parallel readers.""" From 0b9bbf713e6e588817f9d9b1c4c9bb58c4c85da1 Mon Sep 17 00:00:00 2001 From: malteos Date: Tue, 13 Oct 2026 23:08:40 +0200 Subject: [PATCH 55/91] fix(repackage): a writer stopped by a failing source logs that it stopped, not that it failed --- cdx_toolkit/repackage/repackager.py | 5 +++++ tests/repackage/test_producer.py | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cdx_toolkit/repackage/repackager.py b/cdx_toolkit/repackage/repackager.py index 069f51d..7bee60d 100644 --- a/cdx_toolkit/repackage/repackager.py +++ b/cdx_toolkit/repackage/repackager.py @@ -389,6 +389,11 @@ async def write_warc_records( tracker.add(bytes_count=len(item.data), records_count=item.job.records_count) self.log_progress('WARC Writer', tracker) await series.close() + except asyncio.CancelledError: + # stopped by the pipeline: another task (e.g. the source) failed, or the run was cancelled + logger.info('WARC writer stopped, aborting the current WARC file') + await series.abort() + raise except BaseException: logger.error('WARC writer failed, aborting the current WARC file') await series.abort() diff --git a/tests/repackage/test_producer.py b/tests/repackage/test_producer.py index 46e02e8..de3f31d 100644 --- a/tests/repackage/test_producer.py +++ b/tests/repackage/test_producer.py @@ -14,7 +14,7 @@ def iter_range_jobs(self): raise RuntimeError('boom') -def test_failing_source_fails_the_pipeline(tmp_path): +def test_failing_source_fails_the_pipeline(tmp_path, caplog): """A source that raises mid-iteration stops the readers and the writer, the error propagates.""" async def read(self, job): return b'x' @@ -22,6 +22,10 @@ async def read(self, job): repackager = Repackager( source=FailingSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=3, ) - with patch.object(HttpRangeReader, 'read', read), pytest.raises(RuntimeError, match='boom'): + with patch.object(HttpRangeReader, 'read', read), pytest.raises(RuntimeError, match='boom'), \ + caplog.at_level('INFO', logger='cdx_toolkit.repackage.repackager'): repackager.run() assert glob.glob(str(tmp_path / 'out-*')) == [] + # the writer was stopped, it did not fail + assert 'WARC writer failed' not in caplog.text + assert 'WARC writer stopped' in caplog.text From 471d818ddffe96d81cff23a674faf67542201651 Mon Sep 17 00:00:00 2001 From: malteos Date: Wed, 14 Oct 2026 10:42:58 +0200 Subject: [PATCH 56/91] fix(indexes): an --filter the Internet Archive does not support is a usage error before any request CDXFetcher's check raised a ValueError (traceback) only when the captures were read. --- CHANGELOG.md | 2 ++ cdx_toolkit/indexes/backends/cdx_api.py | 10 ++++++++++ tests/unit/test_indexes.py | 12 ++++++++++++ tests/unit/test_usage_errors.py | 2 ++ 4 files changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad21514..512158d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ if WARC files with the same prefix exist already, instead of overwriting them (also with `--wb`, which numbered its files after the existing ones before) + fixed `--to` with a day (e.g. `--to 20240410`), which was padded to the end of the month, and February in leap years + + `cdxt index ia --filter` with a filter the Internet Archive does not support (`=`, `~`, `!=`, `!~`) is a usage error + (exit status 2) before any request, instead of a traceback while the captures are read + fixed DNS errors of hosts that worked before: they are retried again (like connection errors) instead of failing at once + removed the deprecated `cdx_iter` and `cdx_size` scripts + S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual diff --git a/cdx_toolkit/indexes/backends/cdx_api.py b/cdx_toolkit/indexes/backends/cdx_api.py index 9aafce9..1d7a622 100644 --- a/cdx_toolkit/indexes/backends/cdx_api.py +++ b/cdx_toolkit/indexes/backends/cdx_api.py @@ -4,6 +4,7 @@ import cdx_toolkit from cdx_toolkit.commoncrawl import normalize_crawl +from cdx_toolkit.compat import munge_filter from ..base import IndexBackend, QueryPlan, UsageError from ..query import IndexQuery @@ -32,6 +33,15 @@ def validate(self, query: IndexQuery) -> None: raise UsageError('the CDX API needs at least one --url, --hostname or --domain') if query.crawl and self.source != 'cc': raise UsageError('--crawl can only be used with the Common Crawl index (cc)') + for expression in query.filters: + # the check of CDXFetcher, which would only run when the captures are read + try: + munge_filter([expression], self.source) + except ValueError as e: + raise UsageError( + f'--filter {expression!r}: {e} (the Internet Archive supports only FIELD:REGEX and ' + '!FIELD:REGEX, e.g. status:200)' + ) from None def plan(self, query: IndexQuery) -> 'ApiPlan': requests = [ApiRequest(url, api_params(query, match_type)) for url, match_type in query.url_patterns()] diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py index 9b12f90..52b7869 100644 --- a/tests/unit/test_indexes.py +++ b/tests/unit/test_indexes.py @@ -76,6 +76,18 @@ def test_cdx_api_validate(): CdxApiBackend('ia').validate(IndexQuery(urls=['example.com'], crawl='3')) +@pytest.mark.parametrize('expression', ['=status:200', '!=status:200', '~status:2..', '!~status:2..']) +def test_ia_filter_syntax(expression): + """The Internet Archive has no =, ~ filters: a usage error before any request, not a ValueError when the + captures are read.""" + with patch('cdx_toolkit.CDXFetcher') as fetcher, pytest.raises(UsageError, match='ia does not support'): + CdxApiBackend('ia').records(IndexQuery(urls='example.com/*', filters=expression)) + fetcher.assert_not_called() + + CdxApiBackend('ia').validate(IndexQuery(urls='example.com/*', filters=['status:200', '!mime:text/.*'])) + CdxApiBackend('cc').validate(IndexQuery(urls='example.com/*', filters=expression)) + + def test_unsupported_option(): class NoFilterBackend(CdxApiBackend): supported_options = CdxApiBackend.supported_options - {'filters'} diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py index 14980f4..cd73378 100644 --- a/tests/unit/test_usage_errors.py +++ b/tests/unit/test_usage_errors.py @@ -35,6 +35,8 @@ def test_python_m_cdx_toolkit_cli(monkeypatch, capsys): (['index', CDX_PATH, '--url', 'ex*ample.com/*'], '* can only be at the start or the end'), (['index', 'cc-url-index-s3', '--domain', 'example.com', '--to', '20241332'], "invalid timestamp '20241332'"), (["index", "/data/it's.parquet", '--domain', 'example.com'], 'invalid index location'), + (['index', 'ia', '--url', 'example.com/*', '--filter', '=status:200'], + "--filter '=status:200': ia does not support the filter ="), ], ) def test_usage_errors(capsys, args, error): From 857aa6d74d6d7227ded5169edeb82a6d0fa4ed60 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 09:32:36 +0200 Subject: [PATCH 57/91] docs(readme): fix default limits, --count for CDX files and stale notes; keep the S3 settings --- README.md | 188 +++++++++++++++++++++++++++--------------------------- 1 file changed, 93 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index d32ed9a..252cfb1 100644 --- a/README.md +++ b/README.md @@ -150,15 +150,13 @@ extension but use the CDXJ format (crawl index JSON). Use `--fields url,status,t fields are written. `--count` prints the number of captures instead. For the CDX API this is -a cheap estimate based on the number of index pages; for SQL indexes it is -an exact `count(*)`. +a cheap estimate based on the number of index pages; for CDX files it is an +exact count, and for SQL indexes an exact `count(*)`. -If you don't specify much about the crawls or dates or number of -records you're interested in, some default limits will kick in to -prevent overly-large queries. These default limits include a maximum -of 1000 records (`--limit 1000`) and a limit of 1 year of CC indexes. -To exceed these limits, use `--limit` and `--crawl` or `--from` and -`--to`. +If you don't specify the crawls or dates you're interested in, queries +of `cc` are limited to the CC indexes of the most recent year, to +prevent overly-large queries. To exceed this limit, use `--crawl` or +`--from` and `--to`. There is no default `--limit`. If it seems like nothing is happening, add `-v` or `-vv`: @@ -166,6 +164,85 @@ If it seems like nothing is happening, add `-v` or `-vv`: $ cdxt -vv index cc --url 'commoncrawl.org/*' --count ``` +#### Selecting particular CCF crawls + +Common Crawl's data is divided into "crawls", which were yearly at the +start, and are currently done monthly. There are over 100 of them. +[You can find details about these crawls here.](https://data.commoncrawl.org/crawl-data/index.html) + +Unlike some web archives, CCF doesn't have a single CDX index that +covers all of these crawls -- we have 1 index per crawl. The way +you ask for a particular crawl is: + +``` +$ cdxt index cc --crawl CC-MAIN-2024-33 --url 'commoncrawl.org/*' +``` + +- `--crawl CC-MAIN-2024-33` is a single crawl. +- `--crawl 3` is the latest 3 crawls. +- `--crawl CC-MAIN-2018` will match all of the crawls from 2018. +- `--crawl CC-MAIN-2018,CC-MAIN-2019` will match all of the crawls from 2018 and 2019. + +CCF also has a hive-sharded parquet index (called the columnar index) +that covers all of our crawls. Querying broad time ranges is much +faster with the columnar index. You can find more information about +this index at [the blog post about it](https://commoncrawl.org/blog/index-to-warc-files-and-urls-in-columnar-format). + +The Internet Archive cdx index is organized as a single crawl that goes +from the very beginning until now. That's why there is no `--crawl` for +`ia`, and no default time range: without `--from` and `--to`, a query of +`ia` starts at the beginning of IA's crawl. + +#### Selecting by time + +In most cases you'll probably use --crawl to select the time range for +Common Crawl queries, but for the Internet Archive you'll need to specify +a time range like this: + +``` +$ cdxt index ia --limit 1 --from 2008 --to 200906302359 --url 'commoncrawl.org/*' +``` + +In this example the time range starts at the beginning of 2008 and +ends on June 30, 2009 at 23:59. All times are in UTC. If you do not +specify a time range (and also don't use `--crawl`), `cc` queries get the +most recent year, and `ia` queries all years. + +#### Filter syntax + +Filters can be used to limit captures to a subset of the results. + +Any field name in the output of `cdxt index` can be used in a +filter. These field names are appropriately renamed if the source is +'ia'. The different syntax of filter modifiers for 'ia' and 'cc' is +not fully abstracted away by cdx_toolkit. + +The basic syntax of a filter is `[modifier]field:expression`, for +example `=status:200` or `!=status:200`. + +'cc'-style filters (pywb) come in six flavors: substring match, exact +string, full-match regex, and their inversions. These are indicated by +a modifier of nothing, '=', '\~', '!', '!=', and '!\~', respectively. + +'ia'-style filters (Wayback/OpenWayback) come in two flavors, a full-match +regex and an inverted full-match regex: 'status:200' and '!status:200' + +Multiple filters will be combined with AND. For example, to limit +captures to those which do not have status 200 and do not have status 404, + +``` +$ cdxt index cc --filter '!=status:200' --filter '!=status:404' ... +``` + +Note that filters that discard large numbers of captures put a high +load on the CDX server -- for example, a filter that returns just a +few captures from a domain that has tens of millions of captures is +likely to run very slowly and annoy the owner of the CDX server. + +See https://github.com/webrecorder/pywb/wiki/CDX-Server-API#filter (pywb) +and https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server#filtering (wayback) +for full details of filter modifiers. + ### cdxt repackage `cdxt repackage INDEX_PATH` reads captures written by `cdxt index` (a @@ -249,51 +326,8 @@ All options now go after the command. `iter --csv`/`--jsonl` became gone), `--get` is no longer needed (`--closest` uses a single query), and `--url-fgrep`/`--url-fgrepv` moved from `warc` to `index`. -## Selecting particular CCF crawls - -Common Crawl's data is divided into "crawls", which were yearly at the -start, and are currently done monthly. There are over 100 of them. -[You can find details about these crawls here.](https://data.commoncrawl.org/crawl-data/index.html) - -Unlike some web archives, CCF doesn't have a single CDX index that -covers all of these crawls -- we have 1 index per crawl. The way -you ask for a particular crawl is: - -``` -$ cdxt index cc --crawl CC-MAIN-2024-33 --url 'commoncrawl.org/*' -``` - -- `--crawl CC-MAIN-2024-33` is a single crawl. -- `--crawl 3` is the latest 3 crawls. -- `--crawl CC-MAIN-2018` will match all of the crawls from 2018. -- `--crawl CC-MAIN-2018,CC-MAIN-2019` will match all of the crawls from 2018 and 2019. - -CCF also has a hive-sharded parquet index (called the columnar index) -that covers all of our crawls. Querying broad time ranges is much -faster with the columnar index. You can find more information about -this index at [the blog post about it](https://commoncrawl.org/blog/index-to-warc-files-and-urls-in-columnar-format). - -The Internet Archive cdx index is organized as a single crawl that goes -from the very beginning until now. That's why there is no `--crawl` for -`ia`. Note that cdx queries to `ia` will default to one year year -and limit 1000 entries if you do not specify `--from`, `--to`, and `--limit`. - -## Selecting by time -In most cases you'll probably use --crawl to select the time range for -Common Crawl queries, but for the Internet Archive you'll need to specify -a time range like this: - -``` -$ cdxt index ia --limit 1 --from 2008 --to 200906302359 --url 'commoncrawl.org/*' -``` - -In this example the time range starts at the beginning of 2008 and -ends on June 30, 2009 at 23:59. All times are in UTC. If you do not -specify a time range (and also don't use `--crawl`), you'll get the -most recent year. - -## The full syntax for command-line tools +### The full syntax for command-line tools ``` $ cdxt --help @@ -364,40 +398,6 @@ There's a full example of selecting a subset of captures and writing them into a WARC file in [examples/iter-and-warc.py](examples/iter-and-warc.py). (`cdx_toolkit.warc.get_writer()` is deprecated: use `repackage()`.) -## Filter syntax - -Filters can be used to limit captures to a subset of the results. - -Any field name in the output of `cdxt index` can be used in a -filter. These field names are appropriately renamed if the source is -'ia'. The different syntax of filter modifiers for 'ia' and 'cc' is -not fully abstracted away by cdx_toolkit. - -The basic syntax of a filter is `[modifier]field:expression`, for -example `=status:200` or `!=status:200`. - -'cc'-style filters (pywb) come in six flavors: substring match, exact -string, full-match regex, and their inversions. These are indicated by -a modifier of nothing, '=', '\~', '!', '!=', and '!\~', respectively. - -'ia'-style filters (Wayback/OpenWayback) come in two flavors, a full-match -regex and an inverted full-match regex: 'status:200' and '!status:200' - -Multiple filters will be combined with AND. For example, to limit -captures to those which do not have status 200 and do not have status 404, - -``` -$ cdxt index cc --filter '!=status:200' --filter '!=status:404' ... -``` - -Note that filters that discard large numbers of captures put a high -load on the CDX server -- for example, a filter that returns just a -few captures from a domain that has tens of millions of captures is -likely to run very slowly and annoy the owner of the CDX server. - -See https://github.com/webrecorder/pywb/wiki/CDX-Server-API#filter (pywb) -and https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server#filtering (wayback) -for full details of filter modifiers. ## CDX Jargon, Field Names, and such @@ -432,8 +432,8 @@ should be present in all records, if you want to know exactly what it is. The **limit** argument limits how many captures will be returned. To -help users not shoot themselves in the foot, a limit of 1,000 is -applied to --get and .get() calls. +help users not shoot themselves in the foot, a default limit of 1,000 is +applied to `CDXFetcher.get()` calls. A **filter** allows a user to select a subset of CDX records, reducing network traffic between the CDX API server and the user. For example, @@ -447,8 +447,8 @@ status of '-', not 200. CDX API servers support a **paged interface** for efficient access to large sets of URLs. cdx_toolkit iterators always use the paged interface. cdx_toolkit is also polite to CDX servers by being single-threaded and -serial. If it's not fast enough for you, consider downloading Common -Crawl's index files directly. +serial. If it's not fast enough for you, consider querying Common +Crawl's index files directly with `cdxt index` (CDX files or the columnar index). A **digest** is a sha1 checksum of the contents of a capture. The purpose of a digest is to be able to easily figure out if 2 captures @@ -483,11 +483,9 @@ figuring out the charset using an algorithm similar to browsers. WARC generation should do smart(er) things with revisit records. -Right now the CC code selects which monthly CC indices to use based -solely on date ranges. It would be nice to have an alternative so that -a client could iterate against the most recent N CC indices, and -also have the default one-year lookback use an entire monthly index -instead of a partial one. +Without `--crawl`, the CC code selects which monthly CC indices to use +based solely on date ranges. It would be nice to have the default +one-year lookback use an entire monthly index instead of a partial one. ## Status From c08709169d5cefbcf6d6066d0d0e6adca6c7373e Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 10:10:42 +0200 Subject: [PATCH 58/91] refactor(cli): --index-backend cdx-api|cdx-files|duckdb|athena replaces cdxt index --index-format; cdxt repackage --input-format --- CHANGELOG.md | 5 ++-- README.md | 8 +++--- cdx_toolkit/cli/index.py | 12 ++++----- cdx_toolkit/cli/repackage.py | 6 ++--- cdx_toolkit/indexes/resolve.py | 35 +++++++++++++++++---------- cdx_toolkit/repackage/__init__.py | 14 +++++------ docs/cli-migration-guide.md | 2 +- tests/repackage/test_repackage_cli.py | 8 +++--- tests/repackage/test_sql_index_e2e.py | 6 ++--- tests/unit/test_indexes.py | 4 +-- tests/unit/test_sql_indexes.py | 22 +++++++++++------ 11 files changed, 71 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 512158d..eb947a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ + `--cc-sort` is now passed on to the CDX fetcher + the log level variable is `CDXT_LOGLEVEL` (`LOGLEVEL` still works), like cdx_toolkit's other environment variables + `cdxt index` queries Common Crawl's columnar URL index with DuckDB (`cc-url-index-s3`, `cc-url-index-hf`, `*.parquet`) - or AWS Athena (`athena://...`): `--hostname`/`--domain`/`--crawl`/`--limit` or your own `--sql`, cost confirmation (`--yes`), `--dry-run` + or AWS Athena (`athena://...`): `--hostname`/`--domain`/`--crawl`/`--limit` or your own `--sql`, cost confirmation (`--yes`), `--dry-run`; + `--index-backend cdx-api|cdx-files|duckdb|athena` overrides the backend detected from INDEX + `cdxt repackage` copies WARC records byte-for-byte with the async WARC fetcher (from the feat/warc-range-sources branch): reads from https://, s3:// and hf:// (Hugging Face Storage Buckets), writes to local, s3:// or hf://, `--processes`, `--parallel-readers`, `--add-metadata-record`, glob input; output files are now named `-001.warc.gz` @@ -21,7 +22,7 @@ prefixes (`--url-file-type domain` for host lists), one output per CDX file for an `-o` path without extension, `--processes` + SQL indexes: `--url` patterns (matched against `url_surtkey`), `--from`/`--to` (fetch time), `--language` (primary content language, three-letter ISO 639-3 codes) - + `--output-format cdx` / `--index-format cdx` as another name for `cdxj`; clear error for globs over http(s) + + `--output-format cdx` / `--input-format cdx` as another name for `cdxj`; clear error for globs over http(s) + Parquet output for `cdxt index` and Parquet input for `cdxt repackage` (`cdx_toolkit[parquet]`) + `cdxt repackage`: default `--prefix REPACKAGE` (was `TEST`), no WARC file is written without captures, a full URL in `warc_filename`/`filename` is used as it is (the `warc_url` column is gone); stops (exit status 2) diff --git a/README.md b/README.md index 252cfb1..4f594c3 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,9 @@ $ cdxt index ia --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --wb ia -- ### cdxt index -`cdxt index INDEX` takes one of these indexes (the type is detected from INDEX, or set with -`--index-format cdx|duckdb|athena`): +`cdxt index INDEX` takes one of these indexes. How it is queried is detected from INDEX, or set with +`--index-backend cdx-api|cdx-files|duckdb|athena` (e.g. `cdx-files` for a CDX file at an `https://` +URL without file extension): | INDEX | Index | |---|---| @@ -86,7 +87,7 @@ $ cdxt index ia --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --wb ia -- | `https://...` (no file extension) | another CDX API server, e.g. `https://web.archive.org/cdx/search/cdx` | | `cc-url-index-s3` | Common Crawl's [columnar URL index](https://commoncrawl.org/blog/index-to-warc-files-and-urls-in-columnar-format) on S3 (`s3://commoncrawl/cc-index/table/cc-main/warc/`), queried with DuckDB; needs AWS credentials and `cdx_toolkit[duckdb]` | | `cc-url-index-hf` | the same index on Hugging Face (`hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/`, recent crawls only), queried with DuckDB; needs `cdx_toolkit[duckdb,hf]`. Much slower than S3 | -| `*.parquet`, or a hive-partitioned directory with `--index-format duckdb` | another columnar index, local or remote, queried with DuckDB | +| `*.parquet`, or a hive-partitioned directory with `--index-backend duckdb` | another columnar index, local or remote, queried with DuckDB | | `athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://bucket/path/` | the columnar URL index queried with AWS Athena (database `ccindex`, table `ccindex`; add `&table=...` for another table); needs `cdx_toolkit[athena]` | | other paths, e.g. `'s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz'` | CDX files (one capture per line, optionally gzipped), local or remote, a single file or a glob | @@ -250,6 +251,7 @@ local or remote file, a glob like `s3://bucket/indexes/*.cdx.gz`, or stdin if no path or `-` is given) and writes them into new WARC files. The input format (JSONL, CDXJ, CSV or TSV) is detected from the first line, so Common Crawl's CDX files can be used directly; `*.parquet` files are read as Parquet. +Use `--input-format` to set the format instead. The columnar index column names (`warc_filename`, `warc_record_offset`, `warc_record_length`) work too. A `filename` (or `warc_filename`) that is a full URL (`s3://...`, `https://...`, `hf://...`) is used as it is, otherwise diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index 107e660..31d0c12 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -2,7 +2,7 @@ import logging from cdx_toolkit.indexes import IndexQuery, UsageError, open_index -from cdx_toolkit.indexes.resolve import INDEX_FORMATS +from cdx_toolkit.indexes.resolve import INDEX_BACKENDS from cdx_toolkit.record_io import ( FORMAT_CHOICES, canonical_format, format_from_path, import_pyarrow, open_text, write_records, ) @@ -21,7 +21,7 @@ cc-url-index-s3 Common Crawl's columnar URL index on S3, queried with DuckDB cc-url-index-hf Common Crawl's columnar URL index on Hugging Face, queried with DuckDB PATH other columnar (parquet) index: *.parquet files, or a hive-partitioned - directory with --index-format duckdb + directory with --index-backend duckdb athena://... the columnar URL index queried with AWS Athena, e.g. athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://bucket/path/ PATH CDX file(s), local or remote, may be a glob, e.g. @@ -52,10 +52,10 @@ def add_parser(subparsers): add_verbose_arg(parser, subcommand=True) parser.add_argument('index', metavar='INDEX', help='index name or location (see above)') parser.add_argument( - '--index-format', - choices=INDEX_FORMATS, + '--index-backend', + choices=INDEX_BACKENDS, default='auto', - help='type of the index (default: detected from INDEX)', + help='how INDEX is queried (default: detected from INDEX, see above)', ) selection = parser.add_argument_group( @@ -221,7 +221,7 @@ def writes_per_file(args: Namespace, backend) -> bool: def run(args: Namespace, cmdline: str): backend = open_index( - args.index, args.index_format, cc_mirror=args.cc_mirror, cc_sort=args.cc_sort, assume_yes=args.yes + args.index, args.index_backend, cc_mirror=args.cc_mirror, cc_sort=args.cc_sort, assume_yes=args.yes ) query = build_query(args) diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index 4b5954c..a884f3a 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -50,8 +50,8 @@ def add_parser(subparsers): help="index file(s) with captures, local or remote, may be a glob (default: '-' = stdin)", ) parser.add_argument( - '--index-format', choices=('auto',) + FORMAT_CHOICES, default='auto', - help=('format of the index file (default: parquet for *.parquet, otherwise detected from the first line); ' + '--input-format', choices=('auto',) + FORMAT_CHOICES, default='auto', + help=('format of the input file (default: parquet for *.parquet, otherwise detected from the first line); ' "cdx is the same as cdxj: Common Crawl's index files have the .cdx extension but use the CDXJ format " '(crawl index JSON)'), ) @@ -127,7 +127,7 @@ def run(args: Namespace, cmdline: str): operator=args.operator, is_part_of=args.is_part_of, description=args.description or 'warc extraction generated with: ' + cmdline, - index_format=args.index_format, + input_format=args.input_format, ) LOGGER.info('wrote %d of %d captures to WARC files (%d failed) in %.1f seconds', diff --git a/cdx_toolkit/indexes/resolve.py b/cdx_toolkit/indexes/resolve.py index 7e24fd7..3fd35db 100644 --- a/cdx_toolkit/indexes/resolve.py +++ b/cdx_toolkit/indexes/resolve.py @@ -14,7 +14,7 @@ 'cc-url-index-hf': 'hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/', } -INDEX_FORMATS = ('auto', 'cdx', 'athena', 'duckdb') +INDEX_BACKENDS = ('auto', 'cdx-api', 'cdx-files', 'duckdb', 'athena') GLOB_CHARS = ('*', '?', '[') @@ -32,18 +32,20 @@ def is_cdx_api_location(location: str) -> bool: return '.' not in last_segment -def detect_index_format(location: str) -> str: - """Index format from the location only (no network access).""" +def detect_index_backend(location: str) -> str: + """The backend of an index, from its location only (no network access).""" if location in INDEX_ALIASES or location.endswith('.parquet'): return 'duckdb' if location.startswith('athena://'): return 'athena' - return 'cdx' + if is_cdx_api_location(location): + return 'cdx-api' + return 'cdx-files' def open_index( location: str, - index_format: str = 'auto', + backend: str = 'auto', cc_mirror: Optional[str] = None, cc_sort: Optional[str] = None, assume_yes: bool = False, @@ -52,30 +54,37 @@ def open_index( - 'cc', 'ia', or http(s) URLs without a file extension -> CDX API - 'athena://...' -> Athena - - 'cc-url-index-s3', 'cc-url-index-hf', '*.parquet', or any path with --index-format duckdb -> DuckDB + - 'cc-url-index-s3', 'cc-url-index-hf', '*.parquet', or any path with backend 'duckdb' -> DuckDB - anything else (local or remote CDX files, globs) -> CDX files + + `backend` ('cdx-api', 'cdx-files', 'duckdb' or 'athena') overrides the detection, e.g. 'cdx-files' for a CDX + file at an http(s) URL without file extension. """ - if index_format not in INDEX_FORMATS: - raise UsageError(f'unknown index format {index_format!r} (choose from {", ".join(INDEX_FORMATS)})') - if index_format == 'auto': - index_format = detect_index_format(location) + if backend not in INDEX_BACKENDS: + raise UsageError(f'unknown index backend {backend!r} (choose from {", ".join(INDEX_BACKENDS)})') + if backend == 'auto': + backend = detect_index_backend(location) location = INDEX_ALIASES.get(location, location) - if index_format == 'athena': + if backend == 'athena': from .backends.athena import AthenaBackend return AthenaBackend(location, assume_yes=assume_yes) - if index_format == 'duckdb': + if backend == 'duckdb': if location in INDEX_NAMES or is_cdx_api_location(location) or location.startswith('athena://'): raise UsageError(f'{location} cannot be queried with DuckDB; use a path to the parquet index') from .backends.duckdb import DuckDbBackend return DuckDbBackend(location, assume_yes=assume_yes) - if is_cdx_api_location(location): + if backend == 'cdx-api': + if location not in INDEX_NAMES and urlparse(location).scheme not in ('http', 'https'): + raise UsageError(f'{location} is not a CDX API; use cc, ia or the http(s) URL of a CDX API server') return CdxApiBackend(location, cc_mirror=cc_mirror, cc_sort=cc_sort) + if location in INDEX_NAMES or location.startswith('athena://'): + raise UsageError(f'{location} is not a path of CDX files') from .backends.cdx_files import CdxFilesBackend return CdxFilesBackend(location) diff --git a/cdx_toolkit/repackage/__init__.py b/cdx_toolkit/repackage/__init__.py index af89962..e531d06 100644 --- a/cdx_toolkit/repackage/__init__.py +++ b/cdx_toolkit/repackage/__init__.py @@ -37,7 +37,7 @@ def repackage( operator: Optional[str] = None, is_part_of: Optional[str] = None, description: Optional[str] = None, - index_format: Optional[str] = None, + input_format: Optional[str] = None, ) -> RepackageResult: """Write the captures to the WARC files [-]-001.warc.gz, -002, ... (local, s3:// or hf://). @@ -55,7 +55,7 @@ def repackage( sort: sort the captures by WARC file and offset (better read locality, keeps the captures in memory). log_every_n: log the progress every N records (0 = off). creator, operator, is_part_of, description: fields of the warcinfo record. - index_format: format of the index file, if `captures` is a path (default: detected). + input_format: format of the index file, if `captures` is a path (default: detected). Returns the number of captures requested, written, and failed (could not be fetched; the others are written). Raises UsageError for invalid options, and if the output cannot be written.""" @@ -63,27 +63,27 @@ def repackage( # the minimum interval between requests to the wayback machine only holds within a process raise UsageError('--processes cannot be used with --wb') if isinstance(captures, str): - captures = read_index_file(captures, index_format) + captures = read_index_file(captures, input_format) check_output_is_new(prefix, subprefix) info = warcinfo(prefix, subprefix, creator, operator, is_part_of, description) return _run(captures, prefix, subprefix, size, warc_download_prefix, wb, processes, parallel_readers, list(metadata_records), sort, log_every_n, info) -def read_index_file(path: str, index_format: Optional[str] = None) -> Iterable[dict]: +def read_index_file(path: str, input_format: Optional[str] = None) -> Iterable[dict]: from cdx_toolkit.record_io import canonical_format, check_glob, format_from_path, import_pyarrow, read_records - index_format = None if index_format in (None, 'auto') else canonical_format(index_format) + input_format = None if input_format in (None, 'auto') else canonical_format(input_format) try: check_glob(path) except ValueError as e: raise UsageError(str(e)) from None - if index_format == 'parquet' or (index_format is None and format_from_path(path) == 'parquet'): + if input_format == 'parquet' or (input_format is None and format_from_path(path) == 'parquet'): try: import_pyarrow() except RuntimeError as e: raise UsageError(str(e)) from None - return read_records(path, index_format) + return read_records(path, input_format) def check_output_is_new(prefix: str, subprefix: Optional[str]) -> None: diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index 35a4354..0360e26 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -278,7 +278,7 @@ The `repackage` command of the unreleased `feat/warc-range-sources` branch (#73) | Branch | New | |---|---| | `cdxt --crawl C repackage --target-source sql --engine athena --athena-database D --athena-s3-output S3 --hostnames H ...` | `cdxt index 'athena://athena.us-east-1.amazonaws.com/D?S3OutputLocation=S3' --crawl C --hostname H -o captures.csv` + `cdxt repackage captures.csv ...` | -| `... --engine duckdb --duckdb-index-path PATH` | `cdxt index PATH --index-format duckdb ...` (or `cc-url-index-s3`, `cc-url-index-hf`) | +| `... --engine duckdb --duckdb-index-path PATH` | `cdxt index PATH --index-backend duckdb ...` (or `cc-url-index-s3`, `cc-url-index-hf`) | | `--hostnames H1 H2`, `--domains D1 D2` | `--hostname H1 --hostname H2`, `--domain D1 --domain D2` | | `--query SQL`, `--query-file FILE` | `cdxt index ... --sql FILE` (`--sql -` reads stdin) | | — | new for SQL indexes: `--from`/`--to` (`fetch_time`), `--language` (primary content language, ISO 639-3), Parquet output (`-o captures.parquet`, `cdx_toolkit[parquet]`) | diff --git a/tests/repackage/test_repackage_cli.py b/tests/repackage/test_repackage_cli.py index 571a2b2..34c0424 100644 --- a/tests/repackage/test_repackage_cli.py +++ b/tests/repackage/test_repackage_cli.py @@ -126,7 +126,7 @@ def test_repackage_over_http_with_processes(tmpdir, monkeypatch, mock_data_commo @pytest.mark.parametrize('fmt', ['csv', 'tsv', 'jsonl']) -def test_repackage_index_formats(tmpdir, fmt): +def test_repackage_input_formats(tmpdir, fmt): """Captures written by `cdxt index` in any format (here: with the columnar index column names for csv).""" captures = list(read_records(write_captures(tmpdir))) fields = ['url', 'filename', 'offset', 'length'] @@ -140,9 +140,11 @@ def test_repackage_index_formats(tmpdir, fmt): write_records(captures, path, fmt, fields) main(args=['repackage', path, f'--prefix={tmpdir}/TEST']) + main(args=['repackage', path, f'--input-format={fmt}', f'--prefix={tmpdir}/TEST2']) - responses = read_warc(os.path.join(str(tmpdir), 'TEST-001.warc.gz'))['response'] - assert len(responses) == 10 + for prefix in ('TEST', 'TEST2'): + responses = read_warc(os.path.join(str(tmpdir), f'{prefix}-001.warc.gz'))['response'] + assert len(responses) == 10 def test_repackage_subprefix_and_metadata(tmpdir): diff --git a/tests/repackage/test_sql_index_e2e.py b/tests/repackage/test_sql_index_e2e.py index 581da99..4dab0a5 100644 --- a/tests/repackage/test_sql_index_e2e.py +++ b/tests/repackage/test_sql_index_e2e.py @@ -65,7 +65,7 @@ def _index_and_repackage(tmpdir, index_args, host=HOST): @pytest.mark.usefixtures('mocked') def test_index_duckdb_local_e2e(tmpdir, local_url_index): captures = _index_and_repackage( - tmpdir, [str(local_url_index), '--index-format', 'duckdb', '--crawl', LOCAL_INDEX_CRAWL, + tmpdir, [str(local_url_index), '--index-backend', 'duckdb', '--crawl', LOCAL_INDEX_CRAWL, '--hostname', LOCAL_INDEX_HOST], host=LOCAL_INDEX_HOST, ) # all captures of the host in the crawl, not those of the other crawl @@ -77,7 +77,7 @@ def test_index_duckdb_local_e2e(tmpdir, local_url_index): @pytest.mark.usefixtures('mocked') def test_index_duckdb_local_domain_e2e(tmpdir, local_url_index): captures = _index_and_repackage( - tmpdir, [str(local_url_index), '--index-format', 'duckdb', '--crawl', LOCAL_INDEX_CRAWL, + tmpdir, [str(local_url_index), '--index-backend', 'duckdb', '--crawl', LOCAL_INDEX_CRAWL, '--domain', 'missiondefrance.fr', '--limit', '5'], host=LOCAL_INDEX_HOST, ) assert len(captures) == 5 @@ -94,7 +94,7 @@ def test_index_duckdb_local_sql_e2e(tmpdir, local_url_index): WHERE url_host_name = '{LOCAL_INDEX_HOST}' LIMIT 3 """) - captures = _index_and_repackage(tmpdir, [str(local_url_index), '--index-format', 'duckdb', '--sql', sql, + captures = _index_and_repackage(tmpdir, [str(local_url_index), '--index-backend', 'duckdb', '--sql', sql, '--yes'], host=LOCAL_INDEX_HOST) assert len(captures) == 3 assert {c['languages'] for c in captures} == {'fra'} diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py index 52b7869..034f2e0 100644 --- a/tests/unit/test_indexes.py +++ b/tests/unit/test_indexes.py @@ -36,8 +36,8 @@ def test_open_index(): cdx_files = open_index('s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz') assert isinstance(cdx_files, CdxFilesBackend) - with pytest.raises(UsageError, match='unknown index format'): - open_index('cc', index_format='parquet') + with pytest.raises(UsageError, match='unknown index backend'): + open_index('cc', backend='parquet') def test_query_options_used(): diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index dcd97f0..c70b9b4 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -20,7 +20,7 @@ @pytest.mark.parametrize( - 'location, index_format, backend, backend_location', + 'location, backend_name, backend, backend_location', [ ('cc', 'auto', CdxApiBackend, None), ('cc-url-index-s3', 'auto', DuckDbBackend, 's3://commoncrawl/cc-index/table/cc-main/warc/'), @@ -30,27 +30,33 @@ ('s3://bucket/index/', 'duckdb', DuckDbBackend, 's3://bucket/index/'), (ATHENA, 'auto', AthenaBackend, ATHENA), ('s3://bucket/indexes/cdx-00000.gz', 'auto', CdxFilesBackend, 's3://bucket/indexes/cdx-00000.gz'), - ('./indexes/*.cdx', 'cdx', CdxFilesBackend, './indexes/*.cdx'), + ('./indexes/*.cdx', 'cdx-files', CdxFilesBackend, './indexes/*.cdx'), + ('https://example.com/cdx-00000', 'auto', CdxApiBackend, None), + ('https://example.com/cdx-00000', 'cdx-files', CdxFilesBackend, 'https://example.com/cdx-00000'), + ('https://example.com/cdx.php', 'cdx-api', CdxApiBackend, None), ], ) -def test_resolve_index(location, index_format, backend, backend_location): - resolved = open_index(location, index_format) +def test_resolve_index(location, backend_name, backend, backend_location): + resolved = open_index(location, backend_name) assert isinstance(resolved, backend) if backend_location: assert resolved.location == backend_location @pytest.mark.parametrize( - 'location, index_format, error', + 'location, backend_name, error', [ ('cc', 'duckdb', 'cannot be queried with DuckDB'), ('s3://bucket/index/', 'athena', 'invalid Athena location'), ('athena://athena.us-east-1.amazonaws.com/ccindex', 'auto', 'S3OutputLocation'), + ('./indexes/*.cdx', 'cdx-api', 'is not a CDX API'), + ('cc', 'cdx-files', 'is not a path of CDX files'), + (ATHENA, 'cdx-files', 'is not a path of CDX files'), ], ) -def test_resolve_index_errors(location, index_format, error): +def test_resolve_index_errors(location, backend_name, error): with pytest.raises(UsageError, match=error): - open_index(location, index_format) + open_index(location, backend_name) def test_athena_location(): @@ -212,7 +218,7 @@ def test_index_duckdb_end_to_end(parquet_index, tmp_path, capsys): output = tmp_path / 'out.jsonl' with patch('cdx_toolkit.indexes.backends.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): main([ - 'index', str(parquet_index), '--index-format', 'duckdb', '--domain', 'example.com', + 'index', str(parquet_index), '--index-backend', 'duckdb', '--domain', 'example.com', '--crawl', 'CC-MAIN-2024-33', '-o', str(output), ]) From 116df2ba0941dec22591d83cdd5e790148354f05 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 10:49:29 +0200 Subject: [PATCH 59/91] test: name the open_index() backend tests after open_index --- tests/unit/test_sql_indexes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index c70b9b4..a5e3512 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -36,7 +36,7 @@ ('https://example.com/cdx.php', 'cdx-api', CdxApiBackend, None), ], ) -def test_resolve_index(location, backend_name, backend, backend_location): +def test_open_index_backend(location, backend_name, backend, backend_location): resolved = open_index(location, backend_name) assert isinstance(resolved, backend) if backend_location: @@ -54,7 +54,7 @@ def test_resolve_index(location, backend_name, backend, backend_location): (ATHENA, 'cdx-files', 'is not a path of CDX files'), ], ) -def test_resolve_index_errors(location, backend_name, error): +def test_open_index_backend_errors(location, backend_name, error): with pytest.raises(UsageError, match=error): open_index(location, backend_name) From f2bc3c734bbf6be7598037ae04bea9fc34eaee5e Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 11:17:03 +0200 Subject: [PATCH 60/91] docs: settings.md with the AWS settings and the rate limits and retries of HTTP requests; linked from the README --- README.md | 21 ++++----------- docs/settings.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 docs/settings.md diff --git a/README.md b/README.md index 4f594c3..e4ff48a 100644 --- a/README.md +++ b/README.md @@ -35,22 +35,11 @@ $ pip install cdx_toolkit[parquet] # Parquet output (and input of cdxt repackage $ pip install cdx_toolkit[all] # install all optional dependencies ``` -### S3 settings - -For requests to S3 (e.g. `s3://commoncrawl`), `cdxt` uses Common Crawl's -recommended AWS settings: region `us-east-1`, `retry_mode = adaptive`, -`max_attempts = 100000`, and for S3 `addressing_style = virtual`, -`payload_signing_enabled = false` and `max_concurrent_requests = 2`. -Settings you configured yourself win: environment variables (`AWS_REGION`, -`AWS_DEFAULT_REGION`, `AWS_RETRY_MODE`, `AWS_MAX_ATTEMPTS`), the AWS config file -of your profile (`~/.aws/config`, `AWS_PROFILE`), and options in the fsspec -configuration. `CDXT_AWS_DEFAULTS=0` turns the defaults off. - -The Athena client uses the same retry settings (its region is part of the -`athena://` location). DuckDB has no adaptive retry mode and no limit for -concurrent requests; for S3 it uses the region (`s3_region`), the addressing -style (`s3_url_style`) and `max_attempts` (`http_retries`, with 1 second between -retries). +### Settings + +The AWS settings of S3, Athena and DuckDB requests, and the rate limits and retries of HTTP requests to +Common Crawl and the Internet Archive (with their environment variables), are described in +[docs/settings.md](docs/settings.md). ## Command-line tools diff --git a/docs/settings.md b/docs/settings.md new file mode 100644 index 0000000..8ea85f6 --- /dev/null +++ b/docs/settings.md @@ -0,0 +1,67 @@ +# Settings + +## S3 settings + +For requests to S3 (e.g. `s3://commoncrawl`), `cdxt` uses Common Crawl's +recommended AWS settings: region `us-east-1`, `retry_mode = adaptive`, +`max_attempts = 100000`, and for S3 `addressing_style = virtual`, +`payload_signing_enabled = false` and `max_concurrent_requests = 2`. +Settings you configured yourself win: environment variables (`AWS_REGION`, +`AWS_DEFAULT_REGION`, `AWS_RETRY_MODE`, `AWS_MAX_ATTEMPTS`), the AWS config file +of your profile (`~/.aws/config`, `AWS_PROFILE`), and options in the fsspec +configuration. `CDXT_AWS_DEFAULTS=0` turns the defaults off. + +The Athena client uses the same retry settings (its region is part of the +`athena://` location). DuckDB has no adaptive retry mode and no limit for +concurrent requests; for S3 it uses the region (`s3_region`), the addressing +style (`s3_url_style`) and `max_attempts` (`http_retries`, with 1 second between +retries). + +## HTTP requests: rate limits and retries + +Requests over HTTP(S) go through `cdx_toolkit/myrequests.py`: + +- CDX API queries: `index.commoncrawl.org` for `cc` (and its list of crawls, `collinfo.json`, which is + cached for a day in `CDXT_CACHE_DIR`, default `~/.cache/cdx_toolkit/`), `web.archive.org/cdx/search/cdx` + for `ia`, and other CDX API servers; +- WARC records read over HTTP(S), e.g. from `https://data.commoncrawl.org` (`cdxt repackage` with the + default `--warc-download-prefix`, and `capture.content` in Python); +- wayback replays, e.g. from `https://web.archive.org/web` (`cdxt repackage --wb ia`). + +Reads from `s3://` (see above), `hf://`, and the queries of DuckDB and Athena do not use it. + +### Minimum interval between requests + +Requests to the same host start at least a minimum interval apart, also when they are made from several +threads (e.g. `cdxt repackage --parallel-readers`). After a request, including its retries, the next one +waits the interval again. + +| Host | Interval | Environment variable | +|---|---|---| +| `index.commoncrawl.org` | 1.0 s | `CDXT_CC_INDEX_MIN_RETRY_INTERVAL` | +| `data.commoncrawl.org` | 0.55 s | `CDXT_CC_DATA_MIN_RETRY_INTERVAL` | +| `web.archive.org` (CDX API and replays) | 6.0 s | `CDXT_IA_MIN_RETRY_INTERVAL` | +| any other host | 3.0 s | `CDXT_DEFAULT_MIN_RETRY_INTERVAL` | + +The interval holds within a process: `cdxt repackage --processes N` makes up to N times as many requests +to a host. So reads from `https://data.commoncrawl.org` are limited to at most about 1.8 requests per second +per process, whatever `--parallel-readers` is; for faster reads, use `s3://commoncrawl` in AWS us-east-1. +`--wb` cannot be used with `--processes` for this reason. Please do not lower the intervals for Common +Crawl's and the Internet Archive's servers: they are shared by everybody. + +### Retries + +| Response | What happens | +|---|---| +| status 429, 500, 502, 503, 504, 509 (slow down, temporary errors) | retried without a limit; the wait starts at twice the host's interval and doubles up to 60 s; logged as a warning from the 6th retry | +| connection error, timeout (30 s to connect, 30 s between bytes), broken response | retried after 60 s; a warning from the 11th error (`CDXT_WARNING_AFTER_N_ERRORS`, default 10), fails after 101 errors (`CDXT_MAX_ERRORS`, default 100) | +| unknown hostname (DNS error) | fails at once, unless the host answered before in this process (or is `data.commoncrawl.org`, `commoncrawl.s3.amazonaws.com` or `web.archive.org`): then it is retried like a connection error | +| status 400 or 404 from a CDX API | not an error: no captures, or past the last page | +| status 404 when replaying a capture with status 404, or a revisit (`-`) | not an error: the replay is the 404 | +| other status 400 or 404, and other 4xx/5xx statuses | fails at once, no retry | + +Redirects are not followed. Requests send the `User-Agent` `pypi_cdx_toolkit/`. + +In `cdxt repackage`, a capture whose request fails is counted as failed (the other captures are still +written, see the README). When the run is stopped, the waits end at once; a request in progress ends with +its timeout. From 8d693638f86618c5d6728f61967d8b1100016855 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 11:23:39 +0200 Subject: [PATCH 61/91] reorganized settings.md --- docs/settings.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 8ea85f6..62ee58c 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -1,22 +1,5 @@ # Settings -## S3 settings - -For requests to S3 (e.g. `s3://commoncrawl`), `cdxt` uses Common Crawl's -recommended AWS settings: region `us-east-1`, `retry_mode = adaptive`, -`max_attempts = 100000`, and for S3 `addressing_style = virtual`, -`payload_signing_enabled = false` and `max_concurrent_requests = 2`. -Settings you configured yourself win: environment variables (`AWS_REGION`, -`AWS_DEFAULT_REGION`, `AWS_RETRY_MODE`, `AWS_MAX_ATTEMPTS`), the AWS config file -of your profile (`~/.aws/config`, `AWS_PROFILE`), and options in the fsspec -configuration. `CDXT_AWS_DEFAULTS=0` turns the defaults off. - -The Athena client uses the same retry settings (its region is part of the -`athena://` location). DuckDB has no adaptive retry mode and no limit for -concurrent requests; for S3 it uses the region (`s3_region`), the addressing -style (`s3_url_style`) and `max_attempts` (`http_retries`, with 1 second between -retries). - ## HTTP requests: rate limits and retries Requests over HTTP(S) go through `cdx_toolkit/myrequests.py`: @@ -65,3 +48,20 @@ Redirects are not followed. Requests send the `User-Agent` `pypi_cdx_toolkit/ Date: Thu, 15 Oct 2026 12:11:18 +0200 Subject: [PATCH 62/91] removed notes file (moved to cdxt benchmark repo) --- docs/notes/warc-fetcher-performance.md | 200 ------------------------- 1 file changed, 200 deletions(-) delete mode 100644 docs/notes/warc-fetcher-performance.md diff --git a/docs/notes/warc-fetcher-performance.md b/docs/notes/warc-fetcher-performance.md deleted file mode 100644 index beecf41..0000000 --- a/docs/notes/warc-fetcher-performance.md +++ /dev/null @@ -1,200 +0,0 @@ -# WARC fetcher performance: bottlenecks, analysis & recommendations - -Notes on optimizing the `cdxt repackage` fetch pipeline for large jobs (e.g. a -range-jobs file with ~1M rows), with a focus on running on EC2 in-region against -`s3://commoncrawl`. - -Status legend: **[done]** implemented on this branch · **[proposed]** not yet implemented. - -> These notes were written on the `feat/warc-range-sources` branch (#73). In the new CLI the -> range-job sources became `cdxt index` (CDX API, Athena, DuckDB) and some options were renamed: -> `--parallel`/`--parallel_readers` → `--parallel-readers`, `--no-sort-ranges` → `--no-sort`, -> `--write-paths-as-metadata-records` → `--add-metadata-record`, `--confirm-cost` → `cdxt index --yes`. -> `--hf-reader` was removed: `hf://` is always read with the CDN reader, which now resolves each -> WARC file once instead of once per record. - ---- - -## How the fetcher works today - -Three async stages on a single `asyncio` event loop (`cdx_toolkit/repackage/repackager.py`): - -1. **Produce** — the source (`RecordsSource`, the captures of the index file) yields `RangeJob`s (one per WARC record: - url + byte offset + length). The source is a *sync* generator run in a worker - thread (`asyncio.to_thread(drain)`, `repackager.py`), pushing each job onto a - bounded `range_jobs_queue` via `run_coroutine_threadsafe(...).result()`. -2. **Read** — `num_readers` reader coroutines (`= --parallel`) each pull a job and do - **one S3 `GetObject` (or HTTP GET) per job** (`read_warc_records` → - `RangeReader.read`, `readers/`). The - payload (raw, already-gzipped WARC record bytes) is enqueued. -3. **Write** — `num_writers` (`= num_readers/6`) writer coroutines each own one output - shard and append records to it (`write_warc_records` → `WarcFileSeries.write`). - -Clients/config: - -- One shared `aioboto3` S3 read client and one write client (`get_aws_clients`, - `repackager.py`); read pool sized `num_readers*3`. -- Retries/backoff for throttling (503 SlowDown) in `with_retries` (`retries.py`). -- Output writers: `S3OutputFile` (multipart upload, ≥5 MiB parts) or `LocalOutputFile` - (`aiofiles`); shard files keyed by `writer_id` + `sequence` - (`generate_warc_filename`, `writers/series.py`). Records are written **verbatim** — no - recompression. - ---- - -## Key insight: concurrency vs. cores - -`aioboto3`/`asyncio` give **I/O concurrency**, not **multi-core parallelism**: - -- With `--parallel=30` there are up to 30 S3 requests *in flight at once*; while most - are blocked on the network, others run. For network-bound work this is exactly right. -- But asyncio is single-threaded: **one event loop, one OS thread, one CPU core.** All - the *CPU* work — TLS encrypt/decrypt, HTTP framing, aiobotocore object churn, byte - copies — is serialized on that one core. The readers **and** writers are all - coroutines on the same loop (`asyncio.create_task(...)`, `repackager.py`). -- The GIL means *threads* wouldn't help CPU-bound work either. Only **multiple - processes** use multiple cores. - -Corollary: **sharded output ≠ multi-core.** Having N writers means N concurrent MPU -streams on one core, not N cores. Adding writers (or readers) past the point where the -single core saturates buys nothing but scheduling overhead and memory. - ---- - -## What actually limits a 1M-row job in-region - -**Bandwidth is rarely the constraint.** 1M records at ~30 KB compressed ≈ ~30 GB; over -even 5 Gbps that is ~50 s, at 25 Gbps ~10 s. A c5n NIC is nowhere near the bottleneck. - -**Request rate on a single core usually is.** 1M independent range GETs, each with -per-request TLS + HTTP + Python overhead, all on one event-loop core. A single loop -typically sustains on the order of a few thousand small GET/s before that core -saturates → ~3–8 min CPU-bound on one core, with the NIC mostly idle and the box's other -vCPUs unused. - -Two regimes, and the optimizations differ: - -- **Many small requests** (default): single-core CPU/TLS bound → cut request count - (coalescing), lift the single-core ceiling (uvloop), or use more cores - (multi-process). -- **Fewer, larger reads** (after coalescing, or large records): network-bound → one core - drives multiple Gbps fine; more processes buy little. - -### How to tell which regime you're in - -Run a sample and watch `htop`/`mpstat` during the fetch: - -- One core pinned ~100 %, NIC well below capacity → **single-core CPU-bound** → - coalescing / uvloop / multi-process. -- All cores low, throughput still short → **not CPU-bound**: the limiter is concurrency, - request latency, or S3 throttling — process/writer count is irrelevant. - ---- - -## Writer review (why more output shards won't fix a read-bound core) - -- **No recompression** — records arrive already gzipped and are written verbatim - (`writer.write(item.data)`, `repackager.py`). The writer adds no gzip CPU. -- **`S3OutputFile`** (`writers/s3.py`) buffers into a `bytearray`; on each ≥5 MiB - boundary it does `buffer[:n]` (copy), `del buffer[:n]` (an O(remaining) memmove), - `bytes(chunk)` (another copy), then `await upload_part`. Real but modest byte-copy cost. -- **`LocalOutputFile`** (`writers/local.py`) is `aiofiles` with an 8 KiB buffer and a - `flush()` per write — on a fast read stream the per-flush overhead + EBS limits are the - more likely bottleneck (prefer writing to `s3://` on a c5n). -- **[done] One writer per process.** Because writing is a cheap verbatim copy that one - coroutine sustains, the old `num_writers = num_readers/6` multi-writer fan-out (multiple - shards on one event loop) bought nothing but scheduling overhead and was removed. Each - process now has exactly one writer = one output shard; multi-core scaling and sharding - come from `--processes` (multiple event loops), and shards are merged into one file. The - CPU hot spot is the **read side** (many small GETs + TLS), not the write side. - -The flip side: the shard model makes **multi-process essentially free on the output -side** — shards are independent files with no cross-shard state and no merge step, so N -worker processes can each write their own shards into the same prefix. - ---- - -## Recommendations (ranked) - -Assume parallelism is already configured (`--parallel` high) and reads go to -`s3://commoncrawl` in-region. - -1. **[done] Sort by `(warc_filename, warc_record_offset)`.** Groups records of the same - WARC file with ascending offsets → read locality, connection reuse, and the - prerequisite for coalescing and clean per-file process sharding. Implemented as an - `ORDER BY` on guided SQL queries (Athena/DuckDB) and an in-memory sort when loading a - CSV source; `--no-sort-ranges` opts out. Raw `--query` and `cdx` sources keep their - order. - -2. **[proposed] Coalesce adjacent ranges (highest remaining value).** After sorting, - merge nearby same-file ranges into one GET (fetch the superrange, slice locally by - offset). Safe because each WARC record is an independent gzip member, so concatenated - members remain valid and byte-exact slicing reconstructs each record. Use a gap - threshold (e.g. ≤256 KB–1 MB) to swallow the small request/metadata records between - indexed responses. Effect: collapses ~1M tiny latency/CPU-bound GETs into ~10–100k - larger sequential reads — directly attacks the request-rate ceiling; bandwidth easily - absorbs the gap bytes. Extreme case: a **density heuristic** — when a large fraction of - a ~1 GB file is needed, GET the whole file once and slice. - -3. **[done] Use all cores via multi-process sharding** — confirmed single-core CPU-bound - (a single loop saturates one core at ~450 small GET/s; one process pins ~1 core at - `--parallel 96`). Implemented natively as `cdxt repackage --processes N` - (`cdx_toolkit/repackage/multiprocess.py`): the source is drained once and sharded by - `warc_filename`, one worker process per core fetches its shard - (`--parallel_readers R`, one writer each → one shard/proc), and the shards are merged - into a single `.warc.gz` (`cdx_toolkit/repackage/merge.py`: server-side S3 - `UploadPartCopy`, or fsspec streaming locally). Measured on c5n.xlarge (4 vCPU): - ~457 → ~1130 rec/s fetch (2.6×, cores ~97 %) for a ~1 GiB / 35.7 k-record homepages job; - ~60 s end-to-end including source sharding and the merge. Sublinear because each - process's producer thread (CSV sort + per-row cross-thread enqueue) competes with its - event loop for the GIL — see latent issues. - -4. **[done] uvloop.** Gated by `CDXT_UVLOOP=1` in `Repackager.run` (`repackager.py`). - Measured ~+8 % single-core; stacks under multi-process. - -5. **[proposed] Bound coalesced read size / watch RAM.** A c5n.xlarge has 10.5 GiB; - whole-file (~1 GB) reads × many readers will OOM. Cap superrange size and gap - threshold; revisit `warc_records_queue_size` (`repackager.py`, default 200) since - each buffered payload is larger after coalescing. - -6. **[done] Write output to `s3://` in-region, not local EBS.** Measured: S3-direct MPU - output is within noise of local EBS at this write rate (~35 MB/s total) and skips the - EBS round-trip + separate upload, so prefer `--prefix s3://…`. On the writer count: at - in-region read rates a *single* writer per process keeps up easily (verbatim copy, - 5 MiB parts), so `--parallel_writers 1` is both fastest-enough and gives one shard per - process — better than the `writers = readers/6` default for this workload. - -7. **[proposed] Instance selection follows the constraint.** Because the binding limit is - request-rate/CPU (not bandwidth), **more/faster cores beat more Gbps**. c5n.xlarge's - "up to 25 Gbps" is burstable and depletes to baseline on long jobs anyway — but that - barely matters when 30 GB is seconds of transfer. Scale vCPUs to feed (3); reserve - sustained-25 Gbps types for genuinely bandwidth-bound (huge-record) jobs. - -8. **[proposed] Dedup identical `(filename, offset, length)` rows** before fetching — - free; fold into the sort/coalesce pass. - -### Latent issues worth noting - -- **HTTP path blocks the event loop.** The HTTP read (then in `RangeJob.ranged_get_bytes`) called the - *synchronous* `myrequests_get(...)` without `await`/`to_thread`. - Since the default `--warc-download-prefix` is `https://data.commoncrawl.org`, the - default fetch path serializes the "parallel" readers. On EC2, prefer `s3://commoncrawl` - (the S3 branch is properly async); otherwise the HTTP fetch should be offloaded to a - thread pool / async HTTP client. **[done]**: `HttpRangeReader` (`readers/http.py`) runs it in a thread. -- **Per-row cross-thread enqueue.** The producer does one - `run_coroutine_threadsafe(...).result()` per job (`repackager.py`) — 1M - loop↔thread handoffs. Batching enqueues removes a real per-row overhead at this scale. - **[proposed]** - ---- - -## TL;DR - -1. **[done]** Sort by `(warc_filename, offset)` — locality + enables coalescing. -2. **[done]** Multi-process by filename (`--processes N`) — ~2.6× on 4 cores; shards merged - into one file. The big confirmed win in-region. -3. **[done]** uvloop (`CDXT_UVLOOP=1`), one writer per process, `s3://` output — supporting wins. -4. **[proposed]** Coalesce adjacent ranges, bounded read size, dedup — remaining ideas. - -Always **measure the regime first** (`htop` during a sample run): in-region the limiter -is almost always request-rate on one core, not bandwidth. From f6b3d0ccf04de8c8df784c9fd175dee6143a9e61 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 13:33:34 +0200 Subject: [PATCH 63/91] test: CDXT_LIVE_TESTS=http,s3,athena,hf selects the services to test live; live HF tests; tests use a temporary cache directory CDXT_LIVE_TESTS is a comma-separated list of services (or all): http replaces CDXT_DISABLE_MOCK_RESPONSES, and CDXT_DISABLE_S3_TESTS/CDXT_DISABLE_ATHENA_TESTS are removed. The live S3 and Athena tests read the WARC records from s3://commoncrawl (AWS only). New live tests of cc-url-index-hf and of repackage from/to hf:// buckets (hf://buckets/commoncrawl/commoncrawl-ci-temp, CDXT_TEST_HF_BUCKET). The tests no longer delete the user's cache directory nor write the recorded list of crawls into it. --- .github/workflows/ci-slow.yaml | 6 +- .github/workflows/ci.yaml | 8 -- CONTRIBUTING.md | 18 ++-- tests/conftest.py | 121 ++++++++++++++++++-------- tests/repackage/test_hf_live.py | 101 +++++++++++++++++++++ tests/repackage/test_sql_index_e2e.py | 30 ++++--- 6 files changed, 219 insertions(+), 65 deletions(-) create mode 100644 tests/repackage/test_hf_live.py diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml index ac70953..4d239fe 100644 --- a/.github/workflows/ci-slow.yaml +++ b/.github/workflows/ci-slow.yaml @@ -2,7 +2,8 @@ name: CI-slow on: workflow_dispatch -# live tests (CDXT_LIVE_TESTS=1): S3, Athena and DuckDB against AWS. The permissions are needed to get AWS +# live tests (CDXT_LIVE_TESTS=s3,athena,hf): S3, Athena and DuckDB against AWS, and the Hugging Face Storage +# Buckets (HF_TOKEN: writes to hf://buckets/commoncrawl/commoncrawl-ci-temp). The permissions are needed to get AWS # credentials via GitHub's OIDC token endpoint permissions: id-token: write @@ -58,6 +59,7 @@ jobs: - name: Run tests env: - CDXT_LIVE_TESTS: '1' + CDXT_LIVE_TESTS: 's3,athena,hf' + HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | make test_coverage diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 957a8b9..27479a4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -69,12 +69,6 @@ jobs: - name: Install cdx_toolkit run: pip install .[all] - - name: Disable S3 unit tests for Python 3.8 (botocore requires Python 3.9+) - if: ${{ startsWith(matrix.python-version, '3.8') }} - uses: actions/github-script@v9 - with: - script: | - core.exportVariable('CDXT_DISABLE_S3_TESTS', '1') - name: Set environment variables for faster unit tests (requests are mocked) uses: actions/github-script@v9 with: @@ -85,7 +79,6 @@ jobs: core.exportVariable('CDXT_CC_INDEX_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01') - core.exportVariable('CDXT_DISABLE_ATHENA_TESTS', '1') core.exportVariable('CDXT_LOGLEVEL', 'DEBUG') - name: Lint code @@ -144,7 +137,6 @@ jobs: core.exportVariable('CDXT_CC_INDEX_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01') - core.exportVariable('CDXT_DISABLE_ATHENA_TESTS', '1') core.exportVariable('CDXT_LOGLEVEL', 'DEBUG') - name: test minimal diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7de563..5268bfe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,16 +34,24 @@ make test make unit ``` -By default, all remote requests are mocked: the CDX API and HTTP downloads are served from recorded data in `tests/data`, and S3 is a local [moto](https://github.com/getmoto/moto) server (installed with `.[all]`; without it, the S3 tests are skipped). To change this behaviour and actually call remote APIs (if you run this from a whitelisted IP address), the following environment variable can be set: +By default, all remote requests are mocked: the CDX API and HTTP downloads are served from recorded data in `tests/data`, S3 is a local [moto](https://github.com/getmoto/moto) server (installed with `.[all]`; without it, the S3 tests are skipped), and the Hugging Face reads are faked. The mocked requests still wait for the per-host rate limits ([docs/settings.md](docs/settings.md)); the CI shortens them: ```bash -export CDXT_DISABLE_MOCK_RESPONSES=1 +export CDXT_DEFAULT_MIN_RETRY_INTERVAL=0.01 CDXT_CC_INDEX_MIN_RETRY_INTERVAL=0.01 \ + CDXT_CC_DATA_MIN_RETRY_INTERVAL=0.01 CDXT_IA_MIN_RETRY_INTERVAL=0.01 ``` -The S3 tests run against AWS, and the tests that need AWS and cannot be mocked (Athena, DuckDB queries of Common Crawl's index on S3) run at all, only with AWS credentials and: +To test services live instead, set `CDXT_LIVE_TESTS` to a comma-separated list of them, or `all` (unset the short intervals above, the live servers rate-limit): + +| Service | Live tests | +|---|---| +| `http` | the CDX API (index.commoncrawl.org, web.archive.org) and HTTP downloads (data.commoncrawl.org); needs an allowed IP address | +| `s3` | the S3 tests against AWS, and DuckDB queries of Common Crawl's index on S3; needs AWS credentials | +| `athena` | the Athena queries; needs AWS credentials with Athena access | +| `hf` | `cdxt index cc-url-index-hf`, and `cdxt repackage` reading from `hf://buckets/commoncrawl/commoncrawl` and writing to `hf://buckets/commoncrawl/commoncrawl-ci-temp` (`CDXT_TEST_HF_BUCKET`); writing needs a Hugging Face token (`HF_TOKEN` or `hf auth login`) | ```bash -export CDXT_LIVE_TESTS=1 +CDXT_LIVE_TESTS=s3,athena,hf make test ``` The CI workflow of pull requests runs the mocked tests only; the `CI-slow` workflow (started manually) runs the live tests with AWS credentials. @@ -53,7 +61,7 @@ The WARC records that the `cdxt repackage` tests download (from `data.commoncraw If the remote APIs change, new mock data can be semi-automatically collected by setting another environment variable, running corresponding unit tests, and overwriting existing mock data in `tests/data/mock_responses`: ```bash -# set environment variable (CDXT_DISABLE_MOCK_RESPONSES should not be set) +# set environment variable (CDXT_LIVE_TESTS should not have http) export CDXT_SAVE_MOCK_RESPONSES=./tmp/mock_responses # run the test for what mock data should be saved to $CDXT_SAVE_MOCK_RESPONSES//.jsonl diff --git a/tests/conftest.py b/tests/conftest.py index a731d07..2d5cb2a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,11 +2,15 @@ By default, the tests make no remote requests: - the CDX API and HTTP requests (data.commoncrawl.org) are mocked with `responses` and recorded data - (CDXT_DISABLE_MOCK_RESPONSES=1 sends them to the real servers, see mock_response_from_jsonl) + (see mock_response_from_jsonl) - S3 is a local moto server (see the s3_env fixture and requires_aws_s3) +- Hugging Face reads are faked in the tests that need them -CDXT_LIVE_TESTS=1 runs the S3 tests against AWS instead, and enables the tests that need AWS and -cannot be mocked (Athena, DuckDB over s3://commoncrawl; see requires_live_aws). +CDXT_LIVE_TESTS is a comma-separated list of the services to test live instead (see LIVE_SERVICES), or `all`: +- http: the CDX API and HTTP requests go to the real servers +- s3: the S3 tests run against AWS, and the tests of DuckDB over s3://commoncrawl run (requires_live_s3) +- athena: the Athena tests run (requires_aws_athena) +- hf: the tests of the Hugging Face Storage Bucket hf://buckets/commoncrawl/commoncrawl run (requires_live_hf) """ import gzip import json @@ -17,7 +21,6 @@ from pathlib import Path import pytest -from cdx_toolkit.settings import CACHE_DIR try: import boto3 @@ -49,6 +52,12 @@ except ImportError: # pragma: no cover - exercised in minimal installs _HAS_DUCKDB = False +try: + import huggingface_hub # noqa: F401 + _HAS_HF = True +except ImportError: # pragma: no cover - exercised in minimal installs + _HAS_HF = False + try: import aioboto3 # noqa: F401 _HAS_AIOBOTO3 = True @@ -67,7 +76,6 @@ import requests import responses import base64 -import shutil from unittest.mock import patch @@ -76,9 +84,25 @@ TEST_S3_BUCKET = os.environ.get('CDXT_TEST_S3_BUCKET', 'commoncrawl-ci-temp') TEST_ATHENA_S3_LOCATION = 's3://commoncrawl-ci-temp/athena-results/' TEST_ATHENA_DATABASE = 'ccindex' -DISABLE_ATHENA_TESTS = bool(os.environ.get('CDXT_DISABLE_ATHENA_TESTS', False)) -DISABLE_S3_TESTS = bool(os.environ.get('CDXT_DISABLE_S3_TESTS', False)) -LIVE_TESTS = bool(os.environ.get('CDXT_LIVE_TESTS', False)) +# a writable Hugging Face Storage Bucket (namespace/name) for the hf:// output tests (with CDXT_LIVE_TESTS=hf) +TEST_HF_BUCKET = os.environ.get('CDXT_TEST_HF_BUCKET', 'commoncrawl/commoncrawl-ci-temp') + +LIVE_SERVICES = ('http', 's3', 'athena', 'hf') + + +def _live_services(value: str) -> frozenset: + """The services of CDXT_LIVE_TESTS (comma-separated, or `all`).""" + names = {name.strip() for name in value.split(',') if name.strip()} + if 'all' in names: + return frozenset(LIVE_SERVICES) + unknown = names - set(LIVE_SERVICES) + if unknown: + raise ValueError(f'CDXT_LIVE_TESTS: unknown service(s) {", ".join(sorted(unknown))}; ' + f'use a comma-separated list of {", ".join(LIVE_SERVICES)}, or all') + return frozenset(names) + + +LIVE_TESTS = _live_services(os.environ.get('CDXT_LIVE_TESTS', '')) # Cache for AWS S3/Athena access check to avoid repeated network calls _aws_s3_access_cache = None @@ -86,11 +110,15 @@ @pytest.fixture(scope='session', autouse=True) -def cleanup_cache(): - """Delete cache directory before each test to ensure clean state""" - cache_dir = os.path.expanduser(CACHE_DIR) - if os.path.exists(cache_dir): - shutil.rmtree(cache_dir) +def cache_dir(tmp_path_factory): + """An empty cache directory for the tests: the user's cache (CDXT_CACHE_DIR) keeps its list of crawls, and gets + none of the recorded ones. Also set in the environment, for worker processes.""" + cache = str(tmp_path_factory.mktemp('cdxt-cache')) + '/' + with pytest.MonkeyPatch.context() as mp: + mp.setattr('cdx_toolkit.settings.CACHE_DIR', cache) + mp.setattr('cdx_toolkit.commoncrawl.CACHE_DIR', cache) + mp.setenv('CDXT_CACHE_DIR', cache) + yield cache @pytest.fixture(scope='session', autouse=True) @@ -125,11 +153,11 @@ def check_aws_s3_access(): return _aws_s3_access_cache -def requires_live_aws(func): - """Pytest decorator for tests that need AWS itself (not mockable, e.g. DuckDB over s3://commoncrawl): - they only run with CDXT_LIVE_TESTS=1 and AWS S3 access.""" - if not LIVE_TESTS: - return pytest.mark.skip(reason='needs AWS; set CDXT_LIVE_TESTS=1 to run it')(func) +def requires_live_s3(func): + """Pytest decorator for tests that need AWS S3 itself (not mockable, e.g. DuckDB over s3://commoncrawl): + they only run with CDXT_LIVE_TESTS=s3 and AWS S3 access.""" + if 's3' not in LIVE_TESTS: + return pytest.mark.skip(reason='needs AWS S3; set CDXT_LIVE_TESTS=s3 to run it')(func) if not _HAS_BOTOCORE or not _HAS_FSSPEC: return pytest.mark.skip(reason='S3 dependencies are not installed; install cdx_toolkit[s3]')(func) return pytest.mark.usefixtures('live_aws_s3_access')(func) @@ -138,17 +166,15 @@ def requires_live_aws(func): @pytest.fixture def live_aws_s3_access(): """Skip the test without AWS S3 access; checked when a test needs it (not when the tests are collected).""" - if DISABLE_S3_TESTS: - pytest.skip('AWS S3 access is disabled via environment variable.') if not check_aws_s3_access(): pytest.skip('AWS S3 access not available (no credentials or permissions)') def requires_aws_s3(func): """Pytest decorator for S3 tests: they run against a local moto server (the s3_env fixture), or against - AWS with CDXT_LIVE_TESTS=1.""" - if LIVE_TESTS: - return pytest.mark.usefixtures('s3_env')(requires_live_aws(func)) + AWS with CDXT_LIVE_TESTS=s3.""" + if 's3' in LIVE_TESTS: + return pytest.mark.usefixtures('s3_env')(requires_live_s3(func)) if not _HAS_MOTO: return pytest.mark.skip(reason='moto, s3fs and aioboto3 are needed to mock S3; install cdx_toolkit[all]')(func) return pytest.mark.usefixtures('s3_env')(func) @@ -197,9 +223,9 @@ def check_aws_athena_access(): def requires_aws_athena(func): - """Pytest decorator that skips test if AWS Athena access is not available (needs CDXT_LIVE_TESTS=1).""" - if not LIVE_TESTS: - return pytest.mark.skip(reason='needs AWS Athena; set CDXT_LIVE_TESTS=1 to run it')(func) + """Pytest decorator that skips test if AWS Athena access is not available (needs CDXT_LIVE_TESTS=athena).""" + if 'athena' not in LIVE_TESTS: + return pytest.mark.skip(reason='needs AWS Athena; set CDXT_LIVE_TESTS=athena to run it')(func) if not _HAS_BOTO3: return pytest.mark.skipif( True, reason='S3 dependencies are not installed; install cdx_toolkit[s3] to enable Athena tests.' @@ -211,12 +237,20 @@ def requires_aws_athena(func): @pytest.fixture def live_aws_athena_access(): """Skip the test without AWS Athena access; checked when a test needs it (not when the tests are collected).""" - if DISABLE_ATHENA_TESTS: - pytest.skip('AWS Athena access is disabled via environment variable.') if not check_aws_athena_access(): pytest.skip('AWS Athena access not available (no credentials or permissions)') +def requires_live_hf(func): + """Pytest decorator for tests of the Hugging Face Storage Bucket hf://buckets/commoncrawl/commoncrawl: they only + run with CDXT_LIVE_TESTS=hf (public reads need no token, but have lower rate limits without one).""" + if 'hf' not in LIVE_TESTS: + return pytest.mark.skip(reason='needs Hugging Face; set CDXT_LIVE_TESTS=hf to run it')(func) + if not _HAS_HF: + return pytest.mark.skip(reason='huggingface_hub is not installed; install cdx_toolkit[hf]')(func) + return func + + def requires_duckdb(func): """Pytest decorator that skips a test if the optional duckdb dependency is missing.""" return pytest.mark.skipif( @@ -253,9 +287,9 @@ def _data_commoncrawl_range(request): @pytest.fixture def mock_data_commoncrawl(): """WARC record downloads from data.commoncrawl.org are served from recorded data (unless - CDXT_DISABLE_MOCK_RESPONSES=1); other requests made with `requests` fail, unless the test adds mock data (e.g. + CDXT_LIVE_TESTS=http); other requests made with `requests` fail, unless the test adds mock data (e.g. with mock_response_from_jsonl).""" - if os.environ.get('CDXT_DISABLE_MOCK_RESPONSES'): + if 'http' in LIVE_TESTS: yield None return responses.start() @@ -325,7 +359,7 @@ def map(self, fn, items): class S3Env: """S3 for a test: a local moto server with the buckets commoncrawl (the WARC files of WARC_RECORDS) and - TEST_S3_BUCKET, or AWS itself (CDXT_LIVE_TESTS=1).""" + TEST_S3_BUCKET, or AWS itself (CDXT_LIVE_TESTS=s3).""" def __init__(self, offsets=None): # (filename, offset) -> offset in the mocked WARC file, which holds only the recorded records @@ -356,7 +390,7 @@ def moto_server(): def s3_env(request, monkeypatch, tmp_path): """S3 for a test, see S3Env. The moto server is configured with environment variables only, so that also worker processes (cdxt repackage --processes) use it.""" - if LIVE_TESTS: + if 's3' in LIVE_TESTS: yield S3Env() return @@ -425,6 +459,19 @@ def s3_tmpdir(s3_env): pass +@pytest.fixture +def hf_tmpdir(): + """A temporary path in the Hugging Face Storage Bucket TEST_HF_BUCKET (CDXT_LIVE_TESTS=hf), removed afterwards.""" + from huggingface_hub import HfFileSystem + + path = f'buckets/{TEST_HF_BUCKET}/cdx_toolkit/ci/tmpdirs/{uuid.uuid4().hex}' + yield f'hf://{path}' + try: + HfFileSystem().rm(path, recursive=True) + except Exception: # noqa: BLE001 - cleanup only; nothing written, or no permission + pass + + def flexible_param_matcher(expected_params): """Custom matcher that ignores dynamic 'from' parameter timestamps and casts all values to strings""" @@ -472,7 +519,7 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None (if you run this from a whitelisted IP address), the following environment variable can be set: ```bash - export CDXT_DISABLE_MOCK_RESPONSES=1 + export CDXT_LIVE_TESTS=http ``` If the remote APIs change, new mock data can be semi-automatically collected @@ -480,7 +527,7 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None and overwriting existing mock data in `tests/data/mock_responses`: ```bash - # set environment variable (CDXT_DISABLE_MOCK_RESPONSES should not be set) + # set environment variable (CDXT_LIVE_TESTS should not have http) export CDXT_SAVE_MOCK_RESPONSES=./tmp/mock_responses # run the test for what mock data should be saved to $CDXT_SAVE_MOCK_RESPONSES//.jsonl @@ -527,7 +574,7 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None def conditional_mock_responses(func=None, *, auto_mock_data: bool = True): - """Conditionally applies @responses.activate and auto-loads mock data based on CDXT_DISABLE_MOCK_RESPONSES env var. + """Applies @responses.activate and auto-loads mock data, unless CDXT_LIVE_TESTS has http. The mock data is automatically loaded from JSONL file from the tests/data directory and dependinng on the test module and test function. @@ -537,8 +584,8 @@ def conditional_mock_responses(func=None, *, auto_mock_data: bool = True): """ def decorator(f): - # If the flag CDXT_DISABLE_MOCK_RESPONSES is not detected, response mocking remains enabled - if not os.environ.get('CDXT_DISABLE_MOCK_RESPONSES'): + # without CDXT_LIVE_TESTS=http, the responses are mocked + if 'http' not in LIVE_TESTS: # Add responses.activate f = add_mock_responses(f, auto_mock_data=auto_mock_data) diff --git a/tests/repackage/test_hf_live.py b/tests/repackage/test_hf_live.py new file mode 100644 index 0000000..ef1856d --- /dev/null +++ b/tests/repackage/test_hf_live.py @@ -0,0 +1,101 @@ +"""Live tests of the Hugging Face Storage Buckets (CDXT_LIVE_TESTS=hf): `cdxt index cc-url-index-hf`, and +`cdxt repackage` reading from hf://buckets/commoncrawl/commoncrawl and writing to TEST_HF_BUCKET. + +DuckDB reads of the columnar index over hf:// are slow (minutes for a single parquet file, mostly the latency +of the Hub API), so the index test reads a single file with LIMIT, and the repackage tests use known captures. +""" +import json +import os + +import fsspec +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main +from tests.conftest import requires_duckdb, requires_live_hf + + +HF_PREFIX = 'hf://buckets/commoncrawl/commoncrawl' +CRAWL = 'CC-MAIN-2026-17' +INDEX_FILE = (f'{HF_PREFIX}/cc-index/table/cc-main/warc/crawl={CRAWL}/subset=warc/' + 'part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet') + +# captures of CC-MAIN-2026-17 (warc and crawldiagnostics files), from the CDX API +CAPTURES = [ + {'url': 'https://commoncrawl.org/', 'timestamp': '20260410082607', 'status': '200', + 'digest': 'ZVJ34E3JKBK2AXINWBT3QWHBDNYZU32Z', 'offset': '133904328', 'length': '7487', + 'filename': 'crawl-data/CC-MAIN-2026-17/segments/1775805908305.14/warc/' + 'CC-MAIN-20260410081153-20260410111153-00389.warc.gz'}, + {'url': 'https://www.commoncrawl.org/', 'timestamp': '20260413052322', 'status': '200', + 'digest': 'CAZVHFEZWLAZELACXKKCVDGSKTUQCRJ7', 'offset': '597179815', 'length': '7496', + 'filename': 'crawl-data/CC-MAIN-2026-17/segments/1775805908721.20/warc/' + 'CC-MAIN-20260413040425-20260413070425-00340.warc.gz'}, + {'url': 'http://commoncrawl.org/', 'timestamp': '20260410082607', 'status': '301', + 'digest': 'LGGT76I5HYSS6HQT36GPANELE4H7FWR7', 'offset': '157143', 'length': '877', + 'filename': 'crawl-data/CC-MAIN-2026-17/segments/1775805908305.14/crawldiagnostics/' + 'CC-MAIN-20260410081153-20260410111153-00494.warc.gz'}, +] + + +def _write_captures(tmpdir): + path = os.path.join(str(tmpdir), 'captures.jsonl') + with open(path, 'w') as f: + for capture in CAPTURES: + f.write(json.dumps(capture) + '\n') + return path + + +def _responses(path): + """(WARC-Target-URI, WARC-Payload-Digest) of the response records of a WARC file (sorted: the records are + written in the order of their WARC files and offsets).""" + with fsspec.open(path, 'rb') as stream: + return sorted((record.rec_headers.get_header('WARC-Target-URI'), + record.rec_headers.get_header('WARC-Payload-Digest')) + for record in ArchiveIterator(stream) if record.rec_type == 'response') + + +def _expected(): + return sorted((c['url'], 'sha1:' + c['digest']) for c in CAPTURES) + + +@requires_live_hf +@requires_duckdb +def test_index_cc_url_index_hf_sql(tmpdir): + sql = os.path.join(str(tmpdir), 'query.sql') + with open(sql, 'w') as f: + f.write(f""" + SELECT url, warc_filename, warc_record_offset, warc_record_length + FROM read_parquet('{INDEX_FILE}') + LIMIT 3 + """) + captures_path = os.path.join(str(tmpdir), 'captures.jsonl') + main(args=['index', 'cc-url-index-hf', '--sql', sql, '--yes', '-o', captures_path]) + + with open(captures_path) as f: + captures = [json.loads(line) for line in f] + assert len(captures) == 3 + assert all(c['filename'].startswith(f'crawl-data/{CRAWL}/') for c in captures) + assert all(int(c['length']) > 0 for c in captures) + + +@requires_live_hf +def test_repackage_from_hf(tmpdir): + main(args=['repackage', _write_captures(tmpdir), f'--prefix={tmpdir}/TEST_hf', '--warc-download-prefix', + HF_PREFIX]) + + assert _responses(os.path.join(str(tmpdir), 'TEST_hf-001.warc.gz')) == _expected() + + +@requires_live_hf +def test_repackage_to_hf_bucket(tmpdir, hf_tmpdir): + main(args=['repackage', _write_captures(tmpdir), f'--prefix={hf_tmpdir}/TEST_hf', '--warc-download-prefix', + HF_PREFIX]) + + assert _responses(f'{hf_tmpdir}/TEST_hf-001.warc.gz') == _expected() + + +@requires_live_hf +def test_repackage_from_hf_parallel_readers(tmpdir): + main(args=['repackage', _write_captures(tmpdir), f'--prefix={tmpdir}/TEST_hf', '--warc-download-prefix', + HF_PREFIX, '--parallel-readers', '3']) + + assert _responses(os.path.join(str(tmpdir), 'TEST_hf-001.warc.gz')) == _expected() diff --git a/tests/repackage/test_sql_index_e2e.py b/tests/repackage/test_sql_index_e2e.py index 4dab0a5..384080e 100644 --- a/tests/repackage/test_sql_index_e2e.py +++ b/tests/repackage/test_sql_index_e2e.py @@ -1,9 +1,9 @@ """End-to-end tests of the SQL indexes (DuckDB, Athena): `cdxt index ... | cdxt repackage`. By default, DuckDB queries a local columnar index (the local_url_index fixture) and the WARC records are -served by the mocked data.commoncrawl.org (see tests/conftest.py). With CDXT_LIVE_TESTS=1, the live tests -query Common Crawl's index on S3 (DuckDB) or with Athena, only a single crawl partition for a single host -(cheap, partition-pruned -- never an all-crawls scan), and fetch the WARC records over HTTP. +served by the mocked data.commoncrawl.org (see tests/conftest.py). With CDXT_LIVE_TESTS=s3 or athena, the live +tests query Common Crawl's index on S3 (DuckDB) or with Athena, only a single crawl partition for a single host +(cheap, partition-pruned -- never an all-crawls scan), and fetch the WARC records from s3://commoncrawl. """ import json import os @@ -15,12 +15,13 @@ from cdx_toolkit.cli import main from tests.conftest import ( LOCAL_INDEX_CRAWL, LOCAL_INDEX_HOST, TEST_ATHENA_S3_LOCATION, mock_response_from_jsonl, requires_aws_athena, - requires_duckdb, requires_live_aws, + requires_duckdb, requires_live_s3, ) CRAWL = 'CC-MAIN-2026-17' HOST = 'commoncrawl.org' +S3_PREFIX = 's3://commoncrawl' ATHENA = f'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation={TEST_ATHENA_S3_LOCATION}' @@ -37,7 +38,7 @@ def mocked(mock_data_commoncrawl): mock_response_from_jsonl('test_get_cc_endpoints', 'test_cc') -def _index_and_repackage(tmpdir, index_args, host=HOST): +def _index_and_repackage(tmpdir, index_args, host=HOST, warc_download_prefix=None): captures_path = os.path.join(str(tmpdir), 'captures.jsonl') main(args=['index'] + index_args + ['-o', captures_path]) @@ -47,7 +48,8 @@ def _index_and_repackage(tmpdir, index_args, host=HOST): assert all(host in c['url'] for c in captures) base_prefix = str(tmpdir) - main(args=['repackage', captures_path, f'--prefix={base_prefix}/TEST_sql']) + prefix_args = ['--warc-download-prefix', warc_download_prefix] if warc_download_prefix else [] + main(args=['repackage', captures_path, f'--prefix={base_prefix}/TEST_sql'] + prefix_args) response_count = 0 with fsspec.open(os.path.join(base_prefix, 'TEST_sql-001.warc.gz'), 'rb') as stream: @@ -102,22 +104,24 @@ def test_index_duckdb_local_sql_e2e(tmpdir, local_url_index): @requires_aws_athena def test_index_athena_e2e(tmpdir): - _index_and_repackage(tmpdir, [ATHENA, '--crawl', CRAWL, '--hostname', HOST]) + _index_and_repackage(tmpdir, [ATHENA, '--crawl', CRAWL, '--hostname', HOST], warc_download_prefix=S3_PREFIX) -@requires_live_aws +@requires_live_s3 @requires_duckdb def test_index_duckdb_e2e(tmpdir): - _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--hostname', HOST]) + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--hostname', HOST], + warc_download_prefix=S3_PREFIX) -@requires_live_aws +@requires_live_s3 @requires_duckdb def test_index_duckdb_domain_e2e(tmpdir): - _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--domain', HOST, '--limit', '20']) + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--domain', HOST, '--limit', '20'], + warc_download_prefix=S3_PREFIX) -@requires_live_aws +@requires_live_s3 @requires_duckdb def test_index_duckdb_sql_e2e(tmpdir): sql = os.path.join(str(tmpdir), 'query.sql') @@ -128,4 +132,4 @@ def test_index_duckdb_sql_e2e(tmpdir): WHERE url_host_name = '{HOST}' LIMIT 10 """) - _index_and_repackage(tmpdir, ['cc-url-index-s3', '--sql', sql, '--yes']) + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--sql', sql, '--yes'], warc_download_prefix=S3_PREFIX) From 4f004d056b3f06e2256baa494387d9729ecaba47 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 14:10:20 +0200 Subject: [PATCH 64/91] perf(indexes): SQL host selectors use url_host_name_reversed, which follows the sort order of the columnar index --domain matches url_host_name_reversed (the domain or a subdomain) instead of url_surtkey LIKE, with a range that the engines prune row groups by (min/max statistics); --hostname and --url add the range or equality as a hint. url_surtkey is plain-encoded (~22 MB per row group) and LIKE is not pushed down; url_host_name_reversed is dictionary-encoded (~0.2 MB). One file of CC-MAIN-2026-17 over S3: --domain 87 s / 45 MB -> 3 s / 0.8 MB. IP addresses keep the url_surtkey and url_host_name predicates. --- CHANGELOG.md | 3 + cdx_toolkit/indexes/backends/sql_builder.py | 63 ++++++++++++++++++--- tests/conftest.py | 14 +++-- tests/unit/test_selectors.py | 3 +- tests/unit/test_sql_builder.py | 30 ++++++++-- tests/unit/test_sql_indexes.py | 26 ++++++--- 6 files changed, 112 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb947a0..eeca8ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ prefixes (`--url-file-type domain` for host lists), one output per CDX file for an `-o` path without extension, `--processes` + SQL indexes: `--url` patterns (matched against `url_surtkey`), `--from`/`--to` (fetch time), `--language` (primary content language, three-letter ISO 639-3 codes) + + SQL indexes: `--hostname`, `--domain` and `--url` select by `url_host_name_reversed`, which follows the sort order + of the index, so DuckDB (and Athena) skip the row groups of other hosts by their min/max statistics; `--domain` no + longer reads `url_surtkey` (one Parquet file: 3 s instead of 87 s over S3) + `--output-format cdx` / `--input-format cdx` as another name for `cdxj`; clear error for globs over http(s) + Parquet output for `cdxt index` and Parquet input for `cdxt repackage` (`cdx_toolkit[parquet]`) + `cdxt repackage`: default `--prefix REPACKAGE` (was `TEST`), no WARC file is written without captures, diff --git a/cdx_toolkit/indexes/backends/sql_builder.py b/cdx_toolkit/indexes/backends/sql_builder.py index fca0926..7d4fa76 100644 --- a/cdx_toolkit/indexes/backends/sql_builder.py +++ b/cdx_toolkit/indexes/backends/sql_builder.py @@ -55,22 +55,71 @@ def surt_tld(surt: str) -> str: return surt.split(',', 1)[0].split(')', 1)[0] +def reversed_host(host: str) -> Optional[str]: + """The url_host_name_reversed of a host (e.g. www.example.com -> com.example.www), None for an IP address or + a host with other characters than letters, digits, dot and hyphen (these get no url_host_name_reversed + predicates).""" + labels = host.split('.') + if not _SQL_LITERAL_RE.match(host) or not all(labels) or all(label.isdigit() for label in labels): + return None + return '.'.join(reversed(labels)) + + +def reversed_host_range(reversed_prefix: str) -> str: + """url_host_name_reversed values starting with the reversed host: the host and its subdomains (and a few more + hosts, e.g. com.example-shop for com.example). A range, not a LIKE: the engines skip the row groups outside it + by their min/max statistics (the index is sorted by url_surtkey, which follows url_host_name_reversed).""" + escape_sql_literal(reversed_prefix) # validates + return f"url_host_name_reversed >= '{reversed_prefix}' AND url_host_name_reversed < '{reversed_prefix}/'" + + +def surt_host_hint(surt: str) -> Optional[str]: + """The url_host_name_reversed range of the hosts of a SURT key or prefix (e.g. 'com,example)/path' or + 'com,example,'), None if it has no plain host (a port, an IP address).""" + host = surt.split(')', 1)[0].rstrip(',') + reversed_prefix = reversed_host('.'.join(reversed(host.split(',')))) if host else None + return reversed_host_range(reversed_prefix) if reversed_prefix else None + + +def hostname_predicate(hostname: str) -> str: + """Captures of the exact host: url_host_name, and url_host_name_reversed for the min/max statistics (see + reversed_host_range).""" + predicate = f'url_host_name = {escape_sql_literal(hostname)}' + reversed_name = reversed_host(hostname) + if reversed_name is None: + return predicate + return f"(url_host_name_reversed = '{reversed_name}' AND {predicate})" + + def domain_predicate(domain: str) -> str: """Captures of the domain and its subdomains (like --domain for CDX files), e.g. for blog.example.com: - url_host_registered_domain IN ('blog.example.com', 'example.com') - AND (url_surtkey LIKE 'com,example,blog)%' OR url_surtkey LIKE 'com,example,blog,%') + url_host_name_reversed >= 'com.example.blog' AND url_host_name_reversed < 'com.example.blog/' + AND (url_host_name_reversed = 'com.example.blog' OR url_host_name_reversed LIKE 'com.example.blog.%') - The SURT prefixes give the exact host boundary; the registered domain, which is one of the suffixes of the - domain, makes the query cheaper (an equality the engines can push down).""" + The range lets the engines skip the row groups of other hosts (see reversed_host_range), the small + dictionary-encoded url_host_name_reversed gives the exact host boundary. An IP address is matched by its + url_surtkey prefixes and registered domain instead.""" surts = domain_to_surts(domain) labels = surts[0][:-1].split(',')[::-1] # canonical, e.g. without www. + reversed_domain = reversed_host('.'.join(labels)) + if reversed_domain is not None: + return (f'({reversed_host_range(reversed_domain)} AND (url_host_name_reversed = \'{reversed_domain}\' ' + f"OR url_host_name_reversed LIKE '{reversed_domain}.%'))") suffixes = ['.'.join(labels[i:]) for i in range(len(labels) - 1)] or labels registered = ', '.join(escape_sql_literal(suffix) for suffix in suffixes) surt_match = ' OR '.join(f'url_surtkey {sql_like_prefix(surt)}' for surt in surts) return f'(url_host_registered_domain IN ({registered}) AND ({surt_match}))' +def surt_predicate(surt: str, prefix: bool) -> str: + """Captures of a url_surtkey (prefix), with the url_host_name_reversed range of its host for the min/max + statistics (see surt_host_hint).""" + predicate = f'url_surtkey {sql_like_prefix(surt)}' if prefix else f'url_surtkey = {quote_sql_string(surt)}' + hint = surt_host_hint(surt) + return f'({hint} AND {predicate})' if hint else predicate + + def build_where_sql(query: IndexQuery, crawls: Optional[List[str]] = None) -> str: """Build the WHERE body (without the `WHERE` keyword) of a guided query, shared by all SQL engines. @@ -90,10 +139,10 @@ def build_where_sql(query: IndexQuery, crawls: Optional[List[str]] = None) -> st tlds |= {surt_tld(s) for s in surt_prefixes + surts} query_tlds = ' OR '.join(f'url_host_tld = {escape_sql_literal(t)}' for t in sorted(tlds)) - host_predicates = [f'url_host_name = {escape_sql_literal(h)}' for h in hostnames] + host_predicates = [hostname_predicate(h) for h in hostnames] host_predicates += [domain_predicate(d) for d in domains] - host_predicates += [f'url_surtkey = {quote_sql_string(s)}' for s in surts] - host_predicates += [f'url_surtkey {sql_like_prefix(s)}' for s in surt_prefixes] + host_predicates += [surt_predicate(s, prefix=False) for s in surts] + host_predicates += [surt_predicate(s, prefix=True) for s in surt_prefixes] query_hosts = ' OR '.join(host_predicates) clauses = [ diff --git a/tests/conftest.py b/tests/conftest.py index 2d5cb2a..618b66d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -324,15 +324,17 @@ def local_url_index(tmp_path): path = base / f'crawl={crawl}' / 'subset=warc' path.mkdir(parents=True) con.execute('''CREATE OR REPLACE TABLE idx ( - url_surtkey VARCHAR, url VARCHAR, url_host_name VARCHAR, url_host_registered_domain VARCHAR, - url_host_tld VARCHAR, fetch_time TIMESTAMP, fetch_status SMALLINT, content_mime_type VARCHAR, - content_mime_detected VARCHAR, content_digest VARCHAR, content_languages VARCHAR, content_charset VARCHAR, - fetch_redirect VARCHAR, warc_filename VARCHAR, warc_record_offset INTEGER, warc_record_length INTEGER)''') + url_surtkey VARCHAR, url VARCHAR, url_host_name VARCHAR, url_host_name_reversed VARCHAR, + url_host_registered_domain VARCHAR, url_host_tld VARCHAR, fetch_time TIMESTAMP, fetch_status SMALLINT, + content_mime_type VARCHAR, content_mime_detected VARCHAR, content_digest VARCHAR, content_languages VARCHAR, + content_charset VARCHAR, fetch_redirect VARCHAR, warc_filename VARCHAR, warc_record_offset INTEGER, + warc_record_length INTEGER)''') for c in crawl_rows: host = c['url'].split('/')[2] - values = "?, ?, ?, ?, ?, strptime(?, '%Y%m%d%H%M%S'), ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?" + values = "?, ?, ?, ?, ?, ?, strptime(?, '%Y%m%d%H%M%S'), ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?" con.execute(f'INSERT INTO idx VALUES ({values})', [ - c['urlkey'], c['url'], host, '.'.join(host.split('.')[-2:]), host.split('.')[-1], c['timestamp'], + c['urlkey'], c['url'], host, '.'.join(reversed(host.split('.'))), '.'.join(host.split('.')[-2:]), + host.split('.')[-1], c['timestamp'], int(c['status']), c['mime'], c['mime-detected'], c['digest'], c['languages'], c.get('charset'), c['filename'], int(c['offset']), int(c['length']), ]) diff --git a/tests/unit/test_selectors.py b/tests/unit/test_selectors.py index acaba12..6b13196 100644 --- a/tests/unit/test_selectors.py +++ b/tests/unit/test_selectors.py @@ -40,7 +40,8 @@ def indexes(tmp_path_factory): for i, (url, domain) in enumerate(CAPTURES) ) duckdb.connect().execute(f''' - COPY (SELECT *, TIMESTAMP '2024-08-01 12:00:00' AS fetch_time, 'a.warc.gz' AS warc_filename, + COPY (SELECT *, array_to_string(list_reverse(string_split(url_host_name, '.')), '.') AS url_host_name_reversed, + TIMESTAMP '2024-08-01 12:00:00' AS fetch_time, 'a.warc.gz' AS warc_filename, 1 AS warc_record_length, NULL::VARCHAR AS content_mime_type, NULL::VARCHAR AS content_mime_detected, NULL::SMALLINT AS fetch_status, NULL::VARCHAR AS content_digest, NULL::VARCHAR AS content_languages, diff --git a/tests/unit/test_sql_builder.py b/tests/unit/test_sql_builder.py index 04de22a..7020b99 100644 --- a/tests/unit/test_sql_builder.py +++ b/tests/unit/test_sql_builder.py @@ -19,15 +19,17 @@ def test_build_query_hostnames(): def test_build_query_domains_only(): q = build_sql(IndexQuery(domains=['example.com']), 'ccindex') - assert "url_host_registered_domain IN ('example.com')" in q + assert ("(url_host_name_reversed >= 'com.example' AND url_host_name_reversed < 'com.example/' " + "AND (url_host_name_reversed = 'com.example' OR url_host_name_reversed LIKE 'com.example.%'))") in q assert "url_host_tld = 'com'" in q - assert 'url_host_name' not in q + assert 'url_host_name =' not in q + assert 'url_surtkey' not in q.split('WHERE', 1)[1] def test_build_query_hostnames_and_domains(): q = build_sql(IndexQuery(hostnames=['www.example.com'], domains=['example.org']), 'ccindex') - assert "url_host_name = 'www.example.com'" in q - assert "url_host_registered_domain IN ('example.org')" in q + assert "(url_host_name_reversed = 'com.example.www' AND url_host_name = 'www.example.com')" in q + assert "url_host_name_reversed LIKE 'org.example.%'" in q assert "url_host_tld = 'com'" in q assert "url_host_tld = 'org'" in q @@ -86,3 +88,23 @@ def test_resolve_crawl_names(): with patch('cdx_toolkit.indexes.backends.sql_builder.get_cc_endpoints', return_value=endpoints): assert resolve_crawl_names('2') == ['CC-MAIN-2024-33', 'CC-MAIN-2024-38'] assert resolve_crawl_names('CC-MAIN-2024-30') == ['CC-MAIN-2024-30'] + + +def test_build_query_domain_www_is_the_domain(): + # like the SURT keys (and --domain of the CDX files): www. is not part of the domain + q = build_sql(IndexQuery(domains=['www.example.com']), 'ccindex') + assert "url_host_name_reversed = 'com.example' OR url_host_name_reversed LIKE 'com.example.%'" in q + + +def test_build_query_ip_addresses_without_reversed_host(): + q = build_sql(IndexQuery(hostnames=['1.2.3.4'], domains=['10.0.0.1'], urls=['1.2.3.4/x*']), 'ccindex') + assert 'url_host_name_reversed' not in q + assert "url_host_name = '1.2.3.4'" in q + assert "url_host_registered_domain IN ('10.0.0.1', '0.0.1', '0.1')" in q + assert "url_surtkey LIKE '4,3,2,1)/x%'" in q + + +def test_build_query_url_with_host_range(): + q = build_sql(IndexQuery(urls=['www.example.com/blog/*']), 'ccindex') + assert ("(url_host_name_reversed >= 'com.example' AND url_host_name_reversed < 'com.example/' " + "AND url_surtkey LIKE 'com,example)/blog%' ESCAPE '\\')") in q diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index a5e3512..f9135ea 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -117,9 +117,9 @@ def test_duckdb_guided_query_prunes_crawls(): assert plan.n_crawls == 1 assert "read_parquet(['s3://bucket/index/crawl=CC-MAIN-2024-33/subset=warc/*.parquet']" in sql assert 'crawl IN' not in sql # pruned by the glob - assert ("url_host_name = 'www.example.com' OR (url_host_registered_domain IN ('example.org') " - "AND (url_surtkey LIKE 'org,example)%' ESCAPE '\\' " - "OR url_surtkey LIKE 'org,example,%' ESCAPE '\\'))") in sql + assert ("((url_host_name_reversed = 'com.example.www' AND url_host_name = 'www.example.com') " + "OR (url_host_name_reversed >= 'org.example' AND url_host_name_reversed < 'org.example/' " + "AND (url_host_name_reversed = 'org.example' OR url_host_name_reversed LIKE 'org.example.%')))") in sql assert 'url_surtkey, fetch_time, url,' in sql assert sql.rstrip().endswith('LIMIT 5') @@ -129,10 +129,15 @@ def test_guided_query_url_patterns(): sql = AthenaBackend(ATHENA).plan(query).sql assert "(url_host_tld = 'com' OR url_host_tld = 'net' OR url_host_tld = 'org')" in sql - assert ("((url_host_registered_domain IN ('example.com') AND (url_surtkey LIKE 'com,example)%' ESCAPE '\\' " - "OR url_surtkey LIKE 'com,example,%' ESCAPE '\\')) OR url_surtkey = 'org,example)/' " - "OR url_surtkey LIKE 'com,example)/a\\_b%' ESCAPE '\\' " - "OR url_surtkey LIKE 'net,example)%' ESCAPE '\\' OR url_surtkey LIKE 'net,example,%' ESCAPE '\\')") in sql + range_com = "url_host_name_reversed >= 'com.example' AND url_host_name_reversed < 'com.example/'" + range_org = "url_host_name_reversed >= 'org.example' AND url_host_name_reversed < 'org.example/'" + range_net = "url_host_name_reversed >= 'net.example' AND url_host_name_reversed < 'net.example/'" + assert (f"(({range_com} AND (url_host_name_reversed = 'com.example' " + "OR url_host_name_reversed LIKE 'com.example.%')) " + f"OR ({range_org} AND url_surtkey = 'org,example)/') " + f"OR ({range_com} AND url_surtkey LIKE 'com,example)/a\\_b%' ESCAPE '\\') " + f"OR ({range_net} AND url_surtkey LIKE 'net,example)%' ESCAPE '\\') " + f"OR ({range_net} AND url_surtkey LIKE 'net,example,%' ESCAPE '\\'))") in sql def test_guided_query_time_range_and_languages(): @@ -202,7 +207,9 @@ def parquet_index(tmp_path): for r in rows if r[0] == crawl ) con.execute(f''' - COPY (SELECT *, NULL::VARCHAR AS content_mime_detected, NULL::VARCHAR AS content_digest, + COPY (SELECT *, + array_to_string(list_reverse(string_split(url_host_name, '.')), '.') AS url_host_name_reversed, + NULL::VARCHAR AS content_mime_detected, NULL::VARCHAR AS content_digest, NULL::VARCHAR AS content_languages, NULL::VARCHAR AS content_charset, NULL::VARCHAR AS fetch_redirect FROM (VALUES {values}) AS t(url_surtkey, fetch_time, url, content_mime_type, @@ -272,7 +279,8 @@ def test_index_duckdb_time_range_and_language(tmp_path, capsys): for i, (url, t, lang) in enumerate(rows) ) duckdb.connect().execute(f''' - COPY (SELECT *, NULL::VARCHAR AS content_mime_type, + COPY (SELECT *, array_to_string(list_reverse(string_split(url_host_name, '.')), '.') AS url_host_name_reversed, + NULL::VARCHAR AS content_mime_type, NULL::VARCHAR AS content_mime_detected, NULL::SMALLINT AS fetch_status, NULL::VARCHAR AS content_digest, NULL::VARCHAR AS content_charset, NULL::VARCHAR AS fetch_redirect FROM (VALUES {values}) AS t(url, url_surtkey, fetch_time, content_languages, url_host_registered_domain, From 9b6fad688ef88ba39840bd3c1a95b7c548e00cc5 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 16:28:41 +0200 Subject: [PATCH 65/91] test: the Athena access check no longer simulates the IAM policy of a placeholder role, which always skipped the Athena tests --- tests/conftest.py | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 618b66d..9b585e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -180,24 +180,6 @@ def requires_aws_s3(func): return pytest.mark.usefixtures('s3_env')(func) -def check_aws_athena_query_execution_access(): - """Check if AWS Athena StartQueryExecution permission is available.""" - try: - # Use IAM simulation instead of actual query execution - iam_client = boto3.client('iam') - response = iam_client.simulate_principal_policy( - PolicySourceArn=f'arn:aws:sts::{boto3.client("sts").get_caller_identity()["Account"]}:assumed-role/your-role/session', # noqa: E501 - ActionNames=['athena:StartQueryExecution'], - ResourceArns=['*'], - ) - - # Check if access is allowed - return response['EvaluationResults'][0]['EvalDecision'] == 'allowed' - - except (ClientError, NoCredentialsError): - return False - - def check_aws_athena_access(): """Check if AWS Athena access is available.""" global _aws_athena_access_cache @@ -209,13 +191,8 @@ def check_aws_athena_access(): return _aws_athena_access_cache try: - client = boto3.client('athena') - - # Try list databasets - client.list_databases(CatalogName='AwsDataCatalog') - - # Try query access - _aws_athena_access_cache = check_aws_athena_query_execution_access() + boto3.client('athena').list_databases(CatalogName='AwsDataCatalog') + _aws_athena_access_cache = True except (NoCredentialsError, ClientError): _aws_athena_access_cache = False From e253aa7c767f3ef7f1c02cba43d82c193fee8b74 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 16:50:54 +0200 Subject: [PATCH 66/91] ci: CI-slow also runs the HTTP requests live (CDXT_LIVE_TESTS=http,s3,athena,hf) --- .github/workflows/ci-slow.yaml | 9 +++++---- CONTRIBUTING.md | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml index 4d239fe..86d64bd 100644 --- a/.github/workflows/ci-slow.yaml +++ b/.github/workflows/ci-slow.yaml @@ -2,9 +2,10 @@ name: CI-slow on: workflow_dispatch -# live tests (CDXT_LIVE_TESTS=s3,athena,hf): S3, Athena and DuckDB against AWS, and the Hugging Face Storage -# Buckets (HF_TOKEN: writes to hf://buckets/commoncrawl/commoncrawl-ci-temp). The permissions are needed to get AWS -# credentials via GitHub's OIDC token endpoint +# live tests (CDXT_LIVE_TESTS=http,s3,athena,hf): the HTTP requests to the CDX APIs and data.commoncrawl.org, S3, +# Athena and DuckDB against AWS, and the Hugging Face Storage Buckets (HF_TOKEN: writes to +# hf://buckets/commoncrawl/commoncrawl-ci-temp). The permissions are needed to get AWS credentials via GitHub's OIDC +# token endpoint permissions: id-token: write contents: read @@ -59,7 +60,7 @@ jobs: - name: Run tests env: - CDXT_LIVE_TESTS: 's3,athena,hf' + CDXT_LIVE_TESTS: 'http,s3,athena,hf' HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | make test_coverage diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5268bfe..f39690e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ To test services live instead, set `CDXT_LIVE_TESTS` to a comma-separated list o CDXT_LIVE_TESTS=s3,athena,hf make test ``` -The CI workflow of pull requests runs the mocked tests only; the `CI-slow` workflow (started manually) runs the live tests with AWS credentials. +The CI workflow of pull requests runs the mocked tests only; the `CI-slow` workflow (started manually) runs all live tests (`http,s3,athena,hf`) with AWS credentials and `HF_TOKEN`. The WARC records that the `cdxt repackage` tests download (from `data.commoncrawl.org` or `s3://commoncrawl`) are recorded in `tests/data/repackage/warc_records.jsonl`, and DuckDB queries a small local index built from the same captures (the `local_url_index` fixture in `tests/conftest.py`). From 2cbe2103135ba892a8561f35a12b38c9fd75b247 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 16:55:28 +0200 Subject: [PATCH 67/91] test: the tests that can run live are marked live and live_ (pytest -m live) --- CONTRIBUTING.md | 9 +++++++++ tests/conftest.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f39690e..81229e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,6 +54,15 @@ To test services live instead, set `CDXT_LIVE_TESTS` to a comma-separated list o CDXT_LIVE_TESTS=s3,athena,hf make test ``` +The tests that can run live are marked `live` and `live_`, so `-m` selects only them: + +```bash +CDXT_LIVE_TESTS=all pytest -m live tests # all live tests +CDXT_LIVE_TESTS=s3 pytest -m live_s3 tests # only the S3 tests +``` + +The marker only selects the tests; without the service in `CDXT_LIVE_TESTS`, they are skipped (`s3`, `athena`, `hf`) or run with mocked responses (`http`, and `s3` against moto). + The CI workflow of pull requests runs the mocked tests only; the `CI-slow` workflow (started manually) runs all live tests (`http,s3,athena,hf`) with AWS credentials and `HF_TOKEN`. The WARC records that the `cdxt repackage` tests download (from `data.commoncrawl.org` or `s3://commoncrawl`) are recorded in `tests/data/repackage/warc_records.jsonl`, and DuckDB queries a small local index built from the same captures (the `local_url_index` fixture in `tests/conftest.py`). diff --git a/tests/conftest.py b/tests/conftest.py index 9b585e5..195b5b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,10 @@ - s3: the S3 tests run against AWS, and the tests of DuckDB over s3://commoncrawl run (requires_live_s3) - athena: the Athena tests run (requires_aws_athena) - hf: the tests of the Hugging Face Storage Bucket hf://buckets/commoncrawl/commoncrawl run (requires_live_hf) + +The tests that can run live are marked `live` and `live_` (see live_marks), e.g. +`CDXT_LIVE_TESTS=all pytest -m live` runs only the live tests, `CDXT_LIVE_TESTS=s3 pytest -m live_s3` only those of S3. +Without CDXT_LIVE_TESTS, the marked tests are skipped (s3, athena, hf) or run with mocked responses (http, s3). """ import gzip import json @@ -104,6 +108,26 @@ def _live_services(value: str) -> frozenset: LIVE_TESTS = _live_services(os.environ.get('CDXT_LIVE_TESTS', '')) + +def pytest_configure(config): + config.addinivalue_line('markers', 'live: a test that runs live with CDXT_LIVE_TESTS (any service)') + for service in LIVE_SERVICES: + config.addinivalue_line('markers', f'live_{service}: a test that runs live with CDXT_LIVE_TESTS={service}') + + +def live_marks(func, service): + """Marks a test that runs live with CDXT_LIVE_TESTS=: `live` and `live_`.""" + return pytest.mark.live(getattr(pytest.mark, f'live_{service}')(func)) + + +def pytest_collection_modifyitems(config, items): + """The tests with the mocked data.commoncrawl.org make HTTP requests with CDXT_LIVE_TESTS=http.""" + for item in items: + if 'mock_data_commoncrawl' in item.fixturenames and not item.get_closest_marker('live_http'): + item.add_marker(pytest.mark.live) + item.add_marker(pytest.mark.live_http) + + # Cache for AWS S3/Athena access check to avoid repeated network calls _aws_s3_access_cache = None _aws_athena_access_cache = None @@ -156,6 +180,7 @@ def check_aws_s3_access(): def requires_live_s3(func): """Pytest decorator for tests that need AWS S3 itself (not mockable, e.g. DuckDB over s3://commoncrawl): they only run with CDXT_LIVE_TESTS=s3 and AWS S3 access.""" + func = live_marks(func, 's3') if 's3' not in LIVE_TESTS: return pytest.mark.skip(reason='needs AWS S3; set CDXT_LIVE_TESTS=s3 to run it')(func) if not _HAS_BOTOCORE or not _HAS_FSSPEC: @@ -173,6 +198,7 @@ def live_aws_s3_access(): def requires_aws_s3(func): """Pytest decorator for S3 tests: they run against a local moto server (the s3_env fixture), or against AWS with CDXT_LIVE_TESTS=s3.""" + func = live_marks(func, 's3') if 's3' in LIVE_TESTS: return pytest.mark.usefixtures('s3_env')(requires_live_s3(func)) if not _HAS_MOTO: @@ -201,6 +227,7 @@ def check_aws_athena_access(): def requires_aws_athena(func): """Pytest decorator that skips test if AWS Athena access is not available (needs CDXT_LIVE_TESTS=athena).""" + func = live_marks(func, 'athena') if 'athena' not in LIVE_TESTS: return pytest.mark.skip(reason='needs AWS Athena; set CDXT_LIVE_TESTS=athena to run it')(func) if not _HAS_BOTO3: @@ -221,6 +248,7 @@ def live_aws_athena_access(): def requires_live_hf(func): """Pytest decorator for tests of the Hugging Face Storage Bucket hf://buckets/commoncrawl/commoncrawl: they only run with CDXT_LIVE_TESTS=hf (public reads need no token, but have lower rate limits without one).""" + func = live_marks(func, 'hf') if 'hf' not in LIVE_TESTS: return pytest.mark.skip(reason='needs Hugging Face; set CDXT_LIVE_TESTS=hf to run it')(func) if not _HAS_HF: @@ -575,9 +603,9 @@ def wrapper(*args, **kwargs): with patch('requests.get', side_effect=_custom_behavior_with_original(requests.get)): return f(*args, **kwargs) - return wrapper + return live_marks(wrapper, 'http') - return f + return live_marks(f, 'http') if func is None: # Called with arguments: @conditional_mock_responses(auto_mock_data=False) From 329478d18053c6e70af0ef70e98fee3d707a1f3b Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 17:29:19 +0200 Subject: [PATCH 68/91] docs: usage guides for HTTP, AWS (Athena, S3) and Hugging Face, linked from the README; removed the dead link to the moved benchmark notes --- README.md | 9 ++- docs/usage/aws-athena-s3.md | 117 ++++++++++++++++++++++++++++++ docs/usage/http.md | 141 ++++++++++++++++++++++++++++++++++++ docs/usage/huggingface.md | 100 +++++++++++++++++++++++++ 4 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 docs/usage/aws-athena-s3.md create mode 100644 docs/usage/http.md create mode 100644 docs/usage/huggingface.md diff --git a/README.md b/README.md index e4ff48a..056a1cf 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,12 @@ $ cdxt repackage captures.jsonl --prefix COMMONCRAWL $ cdxt index ia --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --wb ia --prefix IA ``` +Usage guides with examples for different setups: + +- [HTTP](docs/usage/http.md): the CDX APIs and Common Crawl's HTTP server, no account needed +- [AWS: Athena and S3](docs/usage/aws-athena-s3.md): the columnar URL index with Athena or DuckDB, and WARC records from `s3://commoncrawl` +- [Hugging Face](docs/usage/huggingface.md): the Storage Bucket `hf://buckets/commoncrawl/commoncrawl`, and HF Jobs + ### cdxt index `cdxt index INDEX` takes one of these indexes. How it is queried is detected from INDEX, or set with @@ -268,8 +274,7 @@ For large jobs, use `--parallel-readers N` (concurrent reads per process) and `--processes N` (e.g. the number of vCPUs). With several processes, the captures are split by WARC file across the processes, which write into one shared `-001`, `-002`, ... series (which process writes which file -depends on timing; each file is a complete WARC). See [docs/notes/warc-fetcher-performance.md](docs/notes/warc-fetcher-performance.md) -for benchmarks and tuning. +depends on timing; each file is a complete WARC). Captures without a WARC location, like Internet Archive captures, are replayed from the wayback machine given with `--wb` (`ia` is a shortcut for diff --git a/docs/usage/aws-athena-s3.md b/docs/usage/aws-athena-s3.md new file mode 100644 index 0000000..98c8bb2 --- /dev/null +++ b/docs/usage/aws-athena-s3.md @@ -0,0 +1,117 @@ +# AWS: Athena and S3 + +Common Crawl's data is in the S3 bucket `s3://commoncrawl` in AWS `us-east-1`: the WARC files, the CDX +files and the [columnar URL index](https://commoncrawl.org/columnar-index). With an AWS account, you can +query the columnar index with [Athena](https://aws.amazon.com/athena/) (serverless SQL, billed per TB +scanned) or with DuckDB, and copy the selected records from S3. This is the fastest setup for large +selections, especially when `cdxt repackage` runs in `us-east-1` (e.g. on EC2), where reads from +`s3://commoncrawl` are fast and free of transfer costs. + +You need: + +- `pip install cdx_toolkit[athena,s3]` (and `[duckdb]` for DuckDB); +- AWS credentials, found like boto3 finds them (environment, `~/.aws` profiles, SSO, instance roles). + Reads from `s3://commoncrawl` need credentials of any AWS account; anonymous requests are refused; +- for Athena: an S3 bucket of your own for the query results (`S3OutputLocation`), and the table of the + columnar index in your account: run the `CREATE EXTERNAL TABLE` and `MSCK REPAIR TABLE` statements of + [cc-index-table](https://github.com/commoncrawl/cc-index-table#query-the-table-in-amazon-athena) once + (database `ccindex`, table `ccindex`), and repeat `MSCK REPAIR TABLE` to add new crawls. + +The region, retries and timeouts of the S3, Athena and DuckDB requests are described in +[settings.md](../settings.md). + +### Query URL Index via Athena + +The Athena INDEX names the region, the database and the result location: + +```sh +ATHENA='athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://my-bucket/athena-results/' +``` + +Add `&table=...` if your table is not called `ccindex`. + +Send a query for a specific host name and a few crawls: + +```sh +cdxt index "$ATHENA" --hostname commoncrawl.org --crawl CC-MAIN-2026-17,CC-MAIN-2026-13 -o captures.csv +``` + +`--dry-run` prints the SQL that `cdxt` generates, and `--count` runs a `count(*)` instead. The query is +restricted to the partitions of the given crawls and to `subset = 'warc'`; `--domain` and `--hostname` use +the column `url_host_name_reversed`, which follows the sort order of the index, so Athena skips most of +the data. With `-v`, `cdxt` logs the amount of data each query scanned and its estimated cost. + +Get all records of a TLD for all crawls with your own SQL (`va.sql`): + +```sql +SELECT url, fetch_time, fetch_status, content_mime_type, crawl, + warc_filename, warc_record_offset, warc_record_length +FROM ccindex +WHERE subset = 'warc' + AND url_host_tld = 'va' +``` + +```sh +cdxt index "$ATHENA" --sql va.sql -o va.csv +``` + +`--sql` runs the query as it is. Without a `crawl` condition, it scans the `url_host_tld` column of +every crawl, so `cdxt` asks for confirmation first (`--yes` skips the question). Try it on one crawl +first (`AND crawl = 'CC-MAIN-2026-17'`) to see the amount of data scanned. For `cdxt repackage`, a query +has to select `url`, `warc_filename`, `warc_record_offset` and `warc_record_length`. + +Get the records of a list of hosts with a table join (like cc-index-table's +[count-domains-alexa-top-1m.sql](https://github.com/commoncrawl/cc-index-table/blob/main/src/sql/examples/cc-index/count-domains-alexa-top-1m.sql)). +Athena joins tables only, so upload the list to S3 and create a table for it once: + +```sh +aws s3 cp hosts.csv s3://my-bucket/hosts/hosts.csv # one host per line, e.g. www.commoncrawl.org + +cat > create-hosts.sql <<'EOF' +CREATE EXTERNAL TABLE IF NOT EXISTS hosts (host string) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION 's3://my-bucket/hosts/' +EOF +aws athena start-query-execution --query-string file://create-hosts.sql \ + --query-execution-context Database=ccindex \ + --result-configuration OutputLocation=s3://my-bucket/athena-results/ +``` + +(or run the statement in the Athena console). Then join it with the index (`hosts.sql`): + +```sql +SELECT cc.url, cc.fetch_time, cc.fetch_status, cc.crawl, + cc.warc_filename, cc.warc_record_offset, cc.warc_record_length +FROM ccindex AS cc +JOIN hosts AS h ON cc.url_host_name = h.host +WHERE cc.crawl = 'CC-MAIN-2026-17' + AND cc.subset = 'warc' +``` + +```sh +cdxt index "$ATHENA" --sql hosts.sql --yes -o hosts-captures.csv +``` + +### Repackage via S3 Bucket + +Read the WARC records from `s3://commoncrawl` and write the WARC files to your bucket: + +```sh +cdxt repackage hosts-captures.csv --warc-download-prefix s3://commoncrawl \ + --prefix s3://my-bucket/warc/HOSTS --size 1GB --processes 8 --parallel-readers 16 +``` + +- `--warc-download-prefix s3://commoncrawl` reads with your credentials; in `us-east-1`, it is much + faster than the default `https://data.commoncrawl.org`, which has a per-process rate limit. +- `--processes` (e.g. the number of vCPUs) and `--parallel-readers` (concurrent reads per process) set + the throughput. The files are named `HOSTS-001.warc.gz`, `-002`, … whatever the number of processes. +- `--prefix` can also be a local path; `--add-metadata-record FILE` adds a file (e.g. the query) as a + metadata record to each WARC file. + +`cdxt index` can write the captures to S3 as well (`-o s3://my-bucket/hosts-captures.csv`), and +`cdxt repackage` can read them from there. + +DuckDB works on the same data without Athena: `cdxt index cc-url-index-s3` queries +`s3://commoncrawl/cc-index/table/cc-main/warc/` with your credentials, with the same options, e.g. +`cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2026-17 -o captures.csv`. It costs +nothing but reads the Parquet footers of every file of the crawls, so it is fastest in `us-east-1`. diff --git a/docs/usage/http.md b/docs/usage/http.md new file mode 100644 index 0000000..a754242 --- /dev/null +++ b/docs/usage/http.md @@ -0,0 +1,141 @@ +# HTTP + +Everything in this guide works without an account or credentials: the indexes and the WARC files are +read over HTTPS from Common Crawl's and the Internet Archive's public servers. + +- Common Crawl's CDX API (`index.commoncrawl.org`, one index per crawl) is described in + [CDXJ index](https://commoncrawl.org/cdxj-index); the WARC files are on `https://data.commoncrawl.org`. +- The Internet Archive's CDX API and replays are described in the + [Wayback Machine APIs](https://archive.org/help/wayback_api.php). + +These servers are shared by everybody, so `cdxt` keeps a minimum interval between requests to each of +them (e.g. 1 s for `index.commoncrawl.org`, 6 s for `web.archive.org`) and retries when they ask it to +slow down; see [settings.md](../settings.md). That makes HTTP the right choice for small selections: +some hosts, URL prefixes or a few crawls. For large selections (a TLD, a long list of hosts), use the +columnar index (below) or, faster, [AWS](aws-athena-s3.md) or [Hugging Face](huggingface.md). + +```sh +pip install cdx_toolkit # CDX API and HTTP downloads +pip install cdx_toolkit[duckdb] # also the columnar index with DuckDB +``` + +## Small scale + +### Query CDX API Server + +All captures of a host in the two most recent crawls (`--crawl 2`; or name them, e.g. +`--crawl CC-MAIN-2026-17,CC-MAIN-2026-13`): + +```sh +cdxt index cc --hostname commoncrawl.org --crawl 2 -o captures.jsonl +``` + +Check the size of a selection first: `--count` is a cheap estimate based on the number of index pages, +and `--dry-run` prints the API requests without sending them: + +```sh +cdxt index cc --hostname commoncrawl.org --crawl 2 --count +cdxt index cc --hostname commoncrawl.org --crawl 2 --dry-run +``` + +Only successful HTML captures, and only some fields (`--filter` uses the CDX API filter syntax, see the +README): + +```sh +cdxt index cc --domain commoncrawl.org --crawl 2 \ + --filter =status:200 --filter '~mime:text/html' \ + --fields url,timestamp,status,mime -o captures.csv +``` + +`--url` takes the CDX API's URL patterns, e.g. `'commoncrawl.org/blog/*'` for a prefix or +`'*.commoncrawl.org'` for a domain. + +The Internet Archive works the same way, with `--from`/`--to` instead of crawls: + +```sh +cdxt index ia --url 'commoncrawl.org/*' --from 2012 --to 2013 --limit 100 -o ia-captures.jsonl +``` + +Write the captures to WARC files. Common Crawl captures are copied byte-for-byte from +`https://data.commoncrawl.org`; Internet Archive captures have no WARC location and are replayed from the +Wayback Machine with `--wb ia`: + +```sh +cdxt repackage captures.jsonl --prefix out/COMMONCRAWL +cdxt repackage ia-captures.jsonl --wb ia --prefix out/IA +``` + +Both commands can also be piped: + +```sh +cdxt index cc --hostname commoncrawl.org --crawl 1 --limit 20 | cdxt repackage --prefix out/CC-SAMPLE +``` + +## Large scale + +### Query URL Index via DuckDB + +The [columnar URL index](https://commoncrawl.org/columnar-index) has one row per capture with columns +such as `url_host_tld`, `url_host_registered_domain`, `content_languages` and the WARC location, in +Parquet files partitioned by `crawl` and `subset` (`warc`, `robotstxt`, `crawldiagnostics`). DuckDB +reads only the columns and row groups a query needs. + +`cdxt index cc-url-index-s3` reads it from `s3://commoncrawl`, which needs AWS credentials (see the +[AWS guide](aws-athena-s3.md)). Without credentials, DuckDB reads the same files over HTTPS. HTTPS can't +list a directory, so the query names the files; every crawl has a list of them in +`crawl-data//cc-index-table.paths.gz`: + +```sh +CRAWL=CC-MAIN-2026-17 +curl -s https://data.commoncrawl.org/crawl-data/$CRAWL/cc-index-table.paths.gz | gunzip \ + | grep '/subset=warc/' | sed "s|^|'https://data.commoncrawl.org/|; s|\$|'|" | paste -sd, - > files.txt +``` + +All captures of a TLD (here `.va`) in that crawl, in `tld.sql`: + +```sh +cat > tld.sql < hosts.sql <-001.warc.gz`, `-002`, …, each about `--size` (default 1 GB), with a +`warcinfo` record that lists the command line. diff --git a/docs/usage/huggingface.md b/docs/usage/huggingface.md new file mode 100644 index 0000000..f12398d --- /dev/null +++ b/docs/usage/huggingface.md @@ -0,0 +1,100 @@ +# Hugging Face + +Recent Common Crawl crawls are also on Hugging Face, in the Storage Bucket +`hf://buckets/commoncrawl/commoncrawl` with the same paths as `s3://commoncrawl`: the WARC files and the +columnar URL index (`cc-index/table/cc-main/warc/`). See +[Getting started with Common Crawl data on Hugging Face](https://commoncrawl.org/blog/getting-started-with-common-crawl-data-on-hugging-face). +`cdxt` can query the index there with DuckDB, read the WARC records from the bucket, write the WARC +files to a bucket of your own, and run as a [Hugging Face Job](https://huggingface.co/docs/huggingface_hub/en/guides/jobs), +next to the data. + +```sh +pip install cdx_toolkit[hf,duckdb] +hf auth login # or set HF_TOKEN +``` + +Reads from the public bucket work without a token, but with lower +[rate limits](https://huggingface.co/docs/hub/en/rate-limits); writing to your bucket needs a token with +write access to it. + +## Query URL index on Hugging Face via DuckDB + +`cc-url-index-hf` takes the same options as `cc-url-index-s3`: + +```sh +cdxt index cc-url-index-hf --hostname commoncrawl.org --crawl CC-MAIN-2026-17 -o captures.jsonl +``` + +This is slow: DuckDB reads the Parquet files through the Hub API, and each of a crawl's 300 files takes +several requests whose latency (and rate limits) add up to many minutes per crawl. Restrict the query +to as few crawls as possible, try it on a single file with `--sql` first: + +```sql +SELECT url, fetch_time, fetch_status, + warc_filename, warc_record_offset, warc_record_length +FROM read_parquet('hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-17/subset=warc/part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet') +WHERE url_host_tld = 'va' +``` + +```sh +cdxt index cc-url-index-hf --sql va.sql --yes -o va.csv +``` + +or run the query as an HF Job (below). For large queries, S3 is faster (see the +[AWS guide](aws-athena-s3.md)). + +## Repackage via HF Bucket + +```sh +cdxt repackage captures.jsonl --warc-download-prefix hf://buckets/commoncrawl/commoncrawl \ + --prefix hf://buckets///warc/CC --parallel-readers 8 +``` + +`cdxt repackage` resolves each WARC file once and then reads its records from the signed download URL, +so the rate-limited Hub API sees one request per WARC file, not per record. `--prefix` can also be a +local path, and `cdxt index` can write to a bucket (`-o hf://buckets///captures.jsonl`). + +## HF Jobs + +An HF Job runs a command on Hugging Face's infrastructure, close to the bucket, without a machine of +your own (it needs a Hugging Face account with Jobs enabled, e.g. PRO or an organization with credits). +`hf jobs uv run` runs a Python script; this one (`cdxt_job.py`) installs `cdx_toolkit` and runs `cdxt` +with the arguments of the job: + +```python +# /// script +# requires-python = ">=3.10" +# dependencies = ["cdx_toolkit[hf,duckdb]"] +# /// +import sys + +from cdx_toolkit.cli import main + +main(args=sys.argv[1:]) +``` + +Two jobs: the first selects the captures and writes them to your bucket, the second writes them to WARC +files in your bucket. `-s HF_TOKEN` passes your token (from the environment or `hf auth login`) to the +job, so it can write to the bucket. + +1) `cdxt index`, writing to an HF bucket: + +```sh +hf jobs uv run -s HF_TOKEN --timeout 2h cdxt_job.py \ + index cc-url-index-hf --hostname commoncrawl.org --crawl CC-MAIN-2026-17 \ + -o hf://buckets///captures.jsonl +``` + +2) `cdxt repackage`: + +```sh +hf jobs uv run -s HF_TOKEN --flavor cpu-upgrade --timeout 2h cdxt_job.py \ + repackage hf://buckets///captures.jsonl \ + --warc-download-prefix hf://buckets/commoncrawl/commoncrawl \ + --prefix hf://buckets///warc/CC --parallel-readers 8 +``` + +`hf jobs uv run` prints the job's logs until it finishes; with `--detach`, it prints the job ID and +returns, and `hf jobs logs ` and `hf jobs ps` follow it. `--timeout` sets the maximum duration, +`--flavor` the hardware (`hf jobs hardware` lists them), and `--namespace` runs the job in an +organization. From 30067d9da2ae0af5a59a451911ad30087c8dcea8 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 17:40:32 +0200 Subject: [PATCH 69/91] fix(index): refuse an output that overwrites an input; one output per CDX file needs a new or empty directory and an -o path without extension - the output was opened before the lazy input, so -o equal to the input (also ./, symlinks, hard links, a glob member, the --url-file, a Parquet index) truncated it; now exit 2 before anything is opened - a rerun into the same directory left the outputs of the earlier run (e.g. a run that selected nothing) - -o out.txt silently wrote one output per file; an unknown extension now needs --output-format --- CHANGELOG.md | 4 +- README.md | 15 ++-- cdx_toolkit/cli/index.py | 25 +++++- cdx_toolkit/indexes/backends/cdx_files.py | 12 ++- cdx_toolkit/indexes/backends/duckdb.py | 3 + cdx_toolkit/indexes/base.py | 6 +- cdx_toolkit/record_io.py | 44 +++++++++++ docs/cli-migration-guide.md | 2 +- tests/unit/test_cdx_files.py | 93 ++++++++++++++++++++++- tests/unit/test_duckdb.py | 16 ++++ tests/unit/test_record_io.py | 45 ++++++++++- 11 files changed, 249 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eeca8ae..45f3722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,8 @@ + `hf://` WARC reads resolve each WARC file once and read its records from the signed CDN URL (one HF Resolvers rate-limit request per WARC file instead of per record) + `cdxt index` reads CDX files (local or remote, globs): `--url`/`--hostname`/`--domain`/`--url-file` matched as SURT - prefixes (`--url-file-type domain` for host lists), one output per CDX file for an `-o` path without extension, - `--processes` + prefixes (`--url-file-type domain` for host lists), one output per CDX file for an `-o` path without extension + (a new or empty directory), `--processes`; `cdxt index` refuses an output that would overwrite its input + SQL indexes: `--url` patterns (matched against `url_surtkey`), `--from`/`--to` (fetch time), `--language` (primary content language, three-letter ISO 639-3 codes) + SQL indexes: `--hostname`, `--domain` and `--url` select by `url_host_name_reversed`, which follows the sort order of the index, so DuckDB (and Athena) skip the row groups of other hosts by their min/max statistics; `--domain` no diff --git a/README.md b/README.md index 056a1cf..7797fc9 100644 --- a/README.md +++ b/README.md @@ -107,11 +107,12 @@ path like `example.com/reports` all URLs starting with it. The patterns are matc so long URL lists are fast. For DuckDB and Athena, join the index with a table of the URLs in a custom `--sql` query instead. -With a glob of CDX files and an `-o` path without file extension, `cdxt index` -writes one output per CDX file, with the same relative path and the selected -lines unchanged (e.g. to filter a whole crawl's index for a URL list), and -`--processes N` filters N files in parallel. No file is written for CDX files -without matches: +With a glob of CDX files and an `-o` path without file extension (or ending +with `/`), `cdxt index` writes one output per CDX file, with the same relative +path and the selected lines unchanged (e.g. to filter a whole crawl's index for +a URL list), and `--processes N` filters N files in parallel. No file is +written for CDX files without matches. The directory has to be new or empty, so +that no output of an earlier run is mixed with this one: ``` $ cdxt index 's3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz' \ @@ -119,6 +120,10 @@ $ cdxt index 's3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx- $ cdxt repackage 's3://my-bucket/eot/CC-MAIN-2024-30/indexes/cdx-*.gz' --prefix EOT-2024 ``` +An `-o` path with an unknown extension (e.g. `out.txt`) needs `--output-format`. +`cdxt index` refuses an output that would overwrite one of the files the query +reads (the CDX or Parquet files, the `--url-file`). + The SQL indexes (DuckDB, Athena) also support `--language CODE` (repeatable): captures whose primary content language (the first of the `languages` field) is one of the given [ISO 639-3](https://iso639-3.sil.org/) codes, e.g. `eng`, diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index 31d0c12..b9cf197 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -1,10 +1,11 @@ from argparse import Namespace, RawDescriptionHelpFormatter import logging +import posixpath from cdx_toolkit.indexes import IndexQuery, UsageError, open_index from cdx_toolkit.indexes.resolve import INDEX_BACKENDS from cdx_toolkit.record_io import ( - FORMAT_CHOICES, canonical_format, format_from_path, import_pyarrow, open_text, write_records, + FORMAT_CHOICES, canonical_format, format_from_path, import_pyarrow, open_text, overlaps, write_records, ) from .common import add_verbose_arg, parse_fields, positive_int @@ -121,7 +122,8 @@ def add_parser(subparsers): output.add_argument( '-o', '--output', default='-', metavar='PATH', help=("output file, local or remote (e.g. s3://...), gzipped if it ends with .gz (default: '-' = stdout); " - 'for CDX files, a path without file extension is a directory with one output per CDX file'), + 'for CDX files, a path without file extension (or ending with /) is a new or empty directory with one ' + 'output per CDX file'), ) output.add_argument( '--output-format', choices=FORMAT_CHOICES, @@ -207,6 +209,14 @@ def get_output_format(args: Namespace) -> str: return fmt +def is_directory_path(path: str) -> bool: + """A path ending with a slash, or whose last component has no file extension (e.g. out/, out, s3://b/out).""" + if path == '-': + return False + path = path.replace('\\', '/') + return path.endswith('/') or not posixpath.splitext(path.rsplit('/', 1)[-1])[1] + + def writes_per_file(args: Namespace, backend) -> bool: """CDX files with an output path without file extension: one output per CDX file.""" from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend @@ -214,16 +224,23 @@ def writes_per_file(args: Namespace, backend) -> bool: return ( isinstance(backend, CdxFilesBackend) and not (args.count or args.dry_run or args.output_format) - and args.output != '-' - and format_from_path(args.output) is None + and is_directory_path(args.output) ) +def check_output_is_not_input(output: str, backend, query: IndexQuery) -> None: + """Writing the output must not overwrite what the query reads (the inputs are read while it is written).""" + for location in backend.input_locations(query): + if overlaps(output, location): + raise UsageError(f'the output {output} would overwrite the input {location}; use another --output') + + def run(args: Namespace, cmdline: str): backend = open_index( args.index, args.index_backend, cc_mirror=args.cc_mirror, cc_sort=args.cc_sort, assume_yes=args.yes ) query = build_query(args) + check_output_is_not_input(args.output, backend, query) if writes_per_file(args, backend): if args.fields: diff --git a/cdx_toolkit/indexes/backends/cdx_files.py b/cdx_toolkit/indexes/backends/cdx_files.py index 4706ab9..8f9dd91 100644 --- a/cdx_toolkit/indexes/backends/cdx_files.py +++ b/cdx_toolkit/indexes/backends/cdx_files.py @@ -12,7 +12,9 @@ from itertools import islice from typing import Iterator, List, Mapping, NamedTuple, Optional, Tuple -from cdx_toolkit.record_io import check_glob, format_from_path, is_glob, open_text, parse_cdxj_line, read_records +from cdx_toolkit.record_io import ( + check_glob, format_from_path, is_empty_dir, is_glob, open_text, parse_cdxj_line, read_records, +) from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up from ..base import IndexBackend, QueryPlan, UsageError @@ -210,6 +212,9 @@ def validate(self, query: IndexQuery) -> None: for pattern in query.urls: url_pattern_to_surts(pattern) + def input_locations(self, query: IndexQuery) -> List[str]: + return [self.location] + ([query.url_file] if query.url_file else []) + def files(self) -> List['CdxFile']: """The CDX files of the location (a file or a glob).""" import fsspec @@ -247,6 +252,11 @@ def write_per_file(self, query: IndexQuery, output_base: str, processes: int = 1 self.validate(query) if query.limit: raise UsageError('--limit cannot be used when writing one output per CDX file') + if not is_empty_dir(output_base): + raise UsageError( + f'the output directory {output_base} is not empty; use a new directory, so that no output of an ' + 'earlier run is mixed with this one' + ) capture_filter = CaptureFilter.from_query(query, url_filters=True) join = (lambda rel: output_base.rstrip('/') + '/' + rel) if '://' in output_base else \ (lambda rel: os.path.join(output_base, *rel.split('/'))) diff --git a/cdx_toolkit/indexes/backends/duckdb.py b/cdx_toolkit/indexes/backends/duckdb.py index 447b8a1..4d7462c 100644 --- a/cdx_toolkit/indexes/backends/duckdb.py +++ b/cdx_toolkit/indexes/backends/duckdb.py @@ -118,5 +118,8 @@ def from_clause(self, crawls): # hive-partitioned index directory: prune the crawls in the glob return build_from_clause(self.location, crawls), True + def input_locations(self, query): + return [self.location] + def iter_sql_rows(self, sql): return iter_duckdb_rows(sql) diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py index f5de51f..d009fa6 100644 --- a/cdx_toolkit/indexes/base.py +++ b/cdx_toolkit/indexes/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from itertools import islice import logging -from typing import FrozenSet, Iterable, Iterator, Mapping +from typing import FrozenSet, Iterable, Iterator, List, Mapping from .query import IndexQuery, field_flag @@ -41,6 +41,10 @@ def validate(self, query: IndexQuery) -> None: if query.limit is not None and query.limit < 1: raise UsageError(f'--limit must be at least 1, not {query.limit}') + def input_locations(self, query: IndexQuery) -> List[str]: + """The files (paths, directories or globs) the query reads, which an output must not overwrite.""" + return [] + @abstractmethod def plan(self, query: IndexQuery) -> QueryPlan: """The query in the form of this backend.""" diff --git a/cdx_toolkit/record_io.py b/cdx_toolkit/record_io.py index 05cb442..72f8296 100644 --- a/cdx_toolkit/record_io.py +++ b/cdx_toolkit/record_io.py @@ -6,14 +6,17 @@ ''' import contextlib import csv +import fnmatch import itertools import json +import os import re import sys import tempfile from typing import IO, Iterable, Iterator, List, Mapping, Optional, Tuple import fsspec +from fsspec.implementations.local import LocalFileSystem TEXT_FORMATS = ('jsonl', 'cdxj', 'csv', 'tsv') @@ -130,6 +133,47 @@ def check_glob(path: str) -> None: ) +def overlaps(output: str, location: str) -> bool: + """Whether writing `output` can overwrite `location` (a file, a directory or a glob), or the other way around: + the same file (also through symlinks or hard links), a file in the directory, or a file matching the glob.""" + if output == '-' or location == '-': + return False + protocols = {fsspec.core.split_protocol(path)[0] or 'file' for path in (output, location)} + if len(protocols) > 1 and protocols != {'file', 'local'}: + return False + out_fs, out_path = fsspec.url_to_fs(output) + in_fs, in_path = fsspec.url_to_fs(location) + if type(out_fs) is not type(in_fs): + return False + if isinstance(out_fs, LocalFileSystem): + out_real = os.path.normcase(os.path.realpath(out_path)) + for path in (in_fs.glob(in_path) if is_glob(in_path) else [in_path]): + real = os.path.normcase(os.path.realpath(path)) + if out_real == real or out_real.startswith(real.rstrip(os.sep) + os.sep) \ + or real.startswith(out_real.rstrip(os.sep) + os.sep): + return True + if os.path.exists(out_path) and os.path.exists(path) and os.path.samefile(out_path, path): + return True + return False + # remote: compare the paths (a glob matches across '/', which rejects rather too much than too little) + out_path, in_path = out_path.rstrip('/'), in_path.rstrip('/') + return ( + out_path == in_path or out_path.startswith(in_path + '/') or in_path.startswith(out_path + '/') + or (is_glob(in_path) and fnmatch.fnmatchcase(out_path, in_path)) + ) + + +def is_empty_dir(path: str) -> bool: + """Whether a local or remote path is missing or an empty directory (for S3, a prefix without objects).""" + fs, fs_path = fsspec.url_to_fs(path) + try: + return not fs.ls(fs_path.rstrip('/'), detail=False) + except FileNotFoundError: + return True + except NotADirectoryError: + return False + + def expand_paths(path: str) -> List[str]: """A local or remote path, or all files matching a glob (e.g. s3://bucket/indexes/*.cdx.gz).""" if not is_glob(path): diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index 0360e26..ff9ac4a 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -296,7 +296,7 @@ The `repackage` command of the unreleased `feat/warc-range-sources` branch (#73) | `--hf-reader fsspec\|cdn` | removed; `hf://` WARCs are always read from the CDN, with one rate-limited `/resolve/` request per WARC file instead of per record (`fsspec` was ~10x slower and dropped records under load in the benchmarks) | | `--processes`, `--warc-download-prefix`, `--prefix`, `--subprefix`, `--size`, `--creator`, `--operator`, `--is-part-of`, `--description` | unchanged (`--size` also accepts e.g. `1GB`) | | `cdxt filter_cdx INPUT_BASE FILTER_FILE OUTPUT_BASE --input-glob GLOB --parallel N` | `cdxt index 'INPUT_BASE/GLOB' --url-file FILTER_FILE -o OUTPUT_BASE --processes N` (one output per CDX file). Add `--url-file-type domain` to match the entries like `--filter-type url` did (`example.com` = the domain with subdomains, entries with a path = URL prefixes); without it, the entries are URL patterns like `--url` | -| `--filter-type surt`, `--overwrite`, `--limit` (per file) | removed; outputs are overwritten, `--limit` only works with a single output | +| `--filter-type surt`, `--overwrite`, `--limit` (per file) | removed; the output directory has to be new or empty, `--limit` only works with a single output | ## Python API diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py index 4fb0085..a409750 100644 --- a/tests/unit/test_cdx_files.py +++ b/tests/unit/test_cdx_files.py @@ -1,5 +1,6 @@ import gzip import json +import os import shutil import pytest @@ -10,7 +11,7 @@ from cdx_toolkit.indexes.patterns import domain_entry_to_surts, url_pattern_to_surts from cdx_toolkit.record_io import write_records -from tests.conftest import TEST_DATA_PATH +from tests.conftest import TEST_DATA_PATH, requires_aws_s3 CDX_PATH = TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz' @@ -223,3 +224,93 @@ def url(line): expected = [line for line in LINES if line.startswith('fr,mnhn,') and '/taxonomy/' in url(line)] assert lines == [line for line in expected if '/term/1' not in url(line)] assert lines + + +@pytest.mark.parametrize('output, inputs', [ + ('in/a.cdx.gz', ['in/a.cdx.gz']), + ('./in/../in/a.cdx.gz', ['in/a.cdx.gz']), + ('hard.cdx.gz', ['in/a.cdx.gz']), + ('link.cdx.gz', ['in/a.cdx.gz']), + ('in/a.cdx.gz', ['in/*.cdx.gz']), + ('in', ['in/*.cdx.gz']), + ('urls.txt', ['in/a.cdx.gz', '--url-file', 'urls.txt']), +]) +def test_output_must_not_overwrite_the_input(tmp_path, monkeypatch, capsys, output, inputs): + monkeypatch.chdir(tmp_path) + (tmp_path / 'in').mkdir() + shutil.copy(CDX_PATH, tmp_path / 'in' / 'a.cdx.gz') + (tmp_path / 'urls.txt').write_text('*.mnhn.fr\n') + os.link(tmp_path / 'in' / 'a.cdx.gz', tmp_path / 'hard.cdx.gz') + try: + os.symlink(tmp_path / 'in' / 'a.cdx.gz', tmp_path / 'link.cdx.gz') + except OSError: # Windows without the privilege + if output == 'link.cdx.gz': + pytest.skip('no symlinks') + before = {p: p.read_bytes() for p in tmp_path.rglob('*') if p.is_file()} + + with pytest.raises(SystemExit) as e: + main(['index'] + inputs + ['--domain', 'mnhn.fr', '-o', output, '--output-format', 'cdxj']) + assert e.value.code == 2 + assert 'would overwrite the input' in capsys.readouterr().err + assert {p: p.read_bytes() for p in tmp_path.rglob('*') if p.is_file()} == before + + +@pytest.mark.parametrize('processes', [1, 2]) +def test_one_output_per_cdx_file_needs_an_empty_directory(tmp_path, capsys, processes): + out = tmp_path / 'out' + out.mkdir() # empty: fine + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '-o', str(out), '--processes', str(processes)]) + output = out / CDX_PATH.name + first = output.read_bytes() + assert first + + # a second run, even one that selects nothing, would leave the outputs of the first run in place + with pytest.raises(SystemExit) as e: + main(['index', str(CDX_PATH), '--domain', 'example.com', '-o', str(out), '--processes', str(processes)]) + assert e.value.code == 2 + assert 'is not empty' in capsys.readouterr().err + assert output.read_bytes() == first + + +@pytest.mark.parametrize('output, per_file', [ + ('out', True), ('out/', True), ('out.v2/', True), ('.out', True), + ('out.cdxj', False), ('out.jsonl.gz', False), +]) +def test_one_output_per_cdx_file_only_without_file_extension(tmp_path, monkeypatch, output, per_file): + monkeypatch.chdir(tmp_path) + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '-o', output]) + assert (tmp_path / output).is_dir() == per_file + + +@pytest.mark.parametrize('output', ['out.txt', 'out.gz', 'out.v2']) +def test_unknown_output_extension(tmp_path, monkeypatch, capsys, output): + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit) as e: + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '-o', output]) + assert e.value.code == 2 + assert 'cannot detect the output format' in capsys.readouterr().err + assert not (tmp_path / output).exists() + + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '-o', output, '--output-format', 'jsonl']) + assert (tmp_path / output).is_file() + + +@requires_aws_s3 +def test_output_checks_on_s3(s3_tmpdir, capsys): + import fsspec + + fs = fsspec.filesystem('s3') + fs.put_file(str(CDX_PATH), f'{s3_tmpdir}/in/a.cdx.gz') + fs.pipe_file(f'{s3_tmpdir}/out/unrelated.txt', b'x') + for output, error in [ + (f'{s3_tmpdir}/in/a.cdx.gz', 'would overwrite the input'), + (f'{s3_tmpdir}/in', 'would overwrite the input'), + (f'{s3_tmpdir}/out', 'is not empty'), + ]: + with pytest.raises(SystemExit): + main(['index', f'{s3_tmpdir}/in/*.cdx.gz', '--domain', 'mnhn.fr', '-o', output]) + assert error in capsys.readouterr().err, output + assert fs.cat_file(f'{s3_tmpdir}/in/a.cdx.gz') == CDX_PATH.read_bytes() + + main(['index', f'{s3_tmpdir}/in/*.cdx.gz', '--domain', 'mnhn.fr', '-o', f'{s3_tmpdir}/out2']) + assert fs.exists(f'{s3_tmpdir}/out2/a.cdx.gz') diff --git a/tests/unit/test_duckdb.py b/tests/unit/test_duckdb.py index 529708f..97e898e 100644 --- a/tests/unit/test_duckdb.py +++ b/tests/unit/test_duckdb.py @@ -94,3 +94,19 @@ def test_connection_setup_and_hf_filesystem(monkeypatch): assert "CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION 'us-east-1');" in statements con.register_filesystem.assert_called_once_with(hf_fs) con.close.assert_called_once() + + +@pytest.mark.parametrize('output', ['index.parquet', 'idx/index.parquet', 'idx/new.parquet']) +def test_output_must_not_overwrite_the_index(tmp_path, monkeypatch, capsys, output): + from cdx_toolkit.cli import main + + monkeypatch.chdir(tmp_path) + (tmp_path / 'idx').mkdir() + for path in ('index.parquet', 'idx/index.parquet'): + (tmp_path / path).write_bytes(b'PAR1') + location = 'idx/' if output.startswith('idx/') else 'index.parquet' + with pytest.raises(SystemExit) as e: + main(['index', location, '--index-backend', 'duckdb', '--domain', 'example.com', '-o', output]) + assert e.value.code == 2 + assert 'would overwrite the input' in capsys.readouterr().err + assert (tmp_path / 'index.parquet').read_bytes() == b'PAR1' diff --git a/tests/unit/test_record_io.py b/tests/unit/test_record_io.py index 53292f7..d85e338 100644 --- a/tests/unit/test_record_io.py +++ b/tests/unit/test_record_io.py @@ -1,9 +1,12 @@ import io import json +import os import pytest -from cdx_toolkit.record_io import RecordWriter, format_from_path, is_glob, read_records, sniff_format, write_records +from cdx_toolkit.record_io import ( + RecordWriter, format_from_path, is_empty_dir, is_glob, overlaps, read_records, sniff_format, write_records, +) RECORDS = [ @@ -188,3 +191,43 @@ def test_csv_of_capture_objects(tmp_path): {'url': 'http://example.com/', 'status': '200', 'redirect': ''}, {'url': 'http://example.com/x', 'status': '200', 'redirect': 'y'}, ] + + +def test_overlaps_local(tmp_path): + (tmp_path / 'in').mkdir() + (tmp_path / 'in' / 'a.cdx').write_text('x\n') + a = str(tmp_path / 'in' / 'a.cdx') + assert overlaps(a, a) + assert overlaps(str(tmp_path / 'in' / '.' / 'a.cdx'), a) + assert overlaps(a, str(tmp_path / 'in' / '*.cdx')) + assert overlaps(str(tmp_path / 'in'), str(tmp_path / 'in' / '*.cdx')) # a directory with the inputs + assert overlaps(str(tmp_path / 'in' / 'b.cdx'), str(tmp_path / 'in')) # a file in the input directory + assert not overlaps(str(tmp_path / 'in' / 'b.cdx'), str(tmp_path / 'in' / '*.cdx')) # no such input (yet) + assert not overlaps(str(tmp_path / 'in2'), str(tmp_path / 'in')) + assert not overlaps('-', a) and not overlaps(a, '-') + assert not overlaps('memory://in/a.cdx', a) + + os.link(a, tmp_path / 'hard.cdx') + assert overlaps(str(tmp_path / 'hard.cdx'), a) + try: + os.symlink(a, tmp_path / 'link.cdx') + except OSError: # Windows without the privilege + return + assert overlaps(str(tmp_path / 'link.cdx'), str(tmp_path / 'in' / '*.cdx')) + + +def test_overlaps_remote(): + assert overlaps('memory://in/a.cdx', 'memory://in/a.cdx') + assert overlaps('memory://in/a.cdx', 'memory://in/*.cdx') + assert overlaps('memory://in/sub/a.cdx', 'memory://in/') + assert overlaps('memory://in', 'memory://in/*.cdx') + assert not overlaps('memory://in/a.jsonl', 'memory://in/*.cdx') + assert not overlaps('memory://in2/a.cdx', 'memory://in/a.cdx') + + +def test_is_empty_dir(tmp_path): + assert is_empty_dir(str(tmp_path / 'missing')) + assert is_empty_dir(str(tmp_path)) + (tmp_path / 'f').write_text('x') + assert not is_empty_dir(str(tmp_path)) + assert not is_empty_dir(str(tmp_path / 'f')) From ad15f4a9add4daae8e847df9c3e385dbce3a12f6 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 17:43:25 +0200 Subject: [PATCH 70/91] fix(indexes): URL prefixes keep a trailing / or ? as a boundary; hostnames, domains and URL patterns include ports - example.com/blog/* selected /blogger and /blog?x (the SURT drops the trailing slash); now like pywb's prefix search range - CDX files: --hostname and --domain missed captures with a port (com,example:8443)/), which SQL selected - a URL pattern with a port and without scheme (example.com:8080/*) became an invalid SURT key - the selector parity tests use SURT keys written out, not computed by the code under test --- cdx_toolkit/indexes/backends/sql_builder.py | 6 +-- cdx_toolkit/indexes/patterns.py | 23 ++++++--- tests/unit/test_cdx_files.py | 9 ++-- tests/unit/test_selectors.py | 57 ++++++++++++++++----- tests/unit/test_sql_builder.py | 2 +- tests/unit/test_sql_indexes.py | 3 +- 6 files changed, 72 insertions(+), 28 deletions(-) diff --git a/cdx_toolkit/indexes/backends/sql_builder.py b/cdx_toolkit/indexes/backends/sql_builder.py index 7d4fa76..6506daf 100644 --- a/cdx_toolkit/indexes/backends/sql_builder.py +++ b/cdx_toolkit/indexes/backends/sql_builder.py @@ -52,7 +52,7 @@ def sql_like_prefix(prefix: str) -> str: def surt_tld(surt: str) -> str: """'com,example)/path' -> 'com'""" - return surt.split(',', 1)[0].split(')', 1)[0] + return re.split('[,):]', surt, maxsplit=1)[0] def reversed_host(host: str) -> Optional[str]: @@ -75,8 +75,8 @@ def reversed_host_range(reversed_prefix: str) -> str: def surt_host_hint(surt: str) -> Optional[str]: """The url_host_name_reversed range of the hosts of a SURT key or prefix (e.g. 'com,example)/path' or - 'com,example,'), None if it has no plain host (a port, an IP address).""" - host = surt.split(')', 1)[0].rstrip(',') + 'com,example,' or 'com,example:8080)/'), None if it has no plain host (an IP address).""" + host = re.split('[):]', surt, maxsplit=1)[0].rstrip(',') reversed_prefix = reversed_host('.'.join(reversed(host.split(',')))) if host else None return reversed_host_range(reversed_prefix) if reversed_prefix else None diff --git a/cdx_toolkit/indexes/patterns.py b/cdx_toolkit/indexes/patterns.py index 5061c14..f502a0d 100644 --- a/cdx_toolkit/indexes/patterns.py +++ b/cdx_toolkit/indexes/patterns.py @@ -10,14 +10,17 @@ def to_surt(url: str) -> str: from url_is_in import convert_url_to_surt + if '://' not in url: + url = 'http://' + url # without a scheme, a port is not recognized ('example.com:8080/') return convert_url_to_surt(url) def url_pattern_to_surts(pattern: str) -> Tuple[List[str], List[str]]: """A CDX API URL pattern -> (SURT prefixes, exact SURT keys). - - `*.example.com`: the domain and its subdomains - - `example.com/*`, `example.com/path*`: all URLs starting with the pattern + - `*.example.com`: the domain and its subdomains (with any port) + - `example.com/*`, `example.com/path*`: all URLs starting with the pattern; a trailing / or ? is kept as a + boundary (`example.com/blog/*` does not select `/blogger` or `/blog?x`), like pywb's prefix search - `example.com/path`: exactly this URL """ pattern = pattern.strip() @@ -26,12 +29,17 @@ def url_pattern_to_surts(pattern: str) -> Tuple[List[str], List[str]]: if not domain or any(c in domain for c in '*/?'): raise UsageError(f'invalid URL pattern {pattern!r}, a domain pattern is e.g. *.example.com') base = to_surt(domain)[:-2] # 'com,example)/' -> 'com,example' - return [base + ')', base + ','], [] + return [base + ')', base + ',', base + ':'], [] # the domain, its subdomains, with a port if pattern.endswith('*'): prefix = pattern[:-1] if '*' in prefix: raise UsageError(f'invalid URL pattern {pattern!r}, * can only be at the start or the end') - return [to_surt(prefix)], [] + key = to_surt(prefix) + # the SURT drops a trailing / or ? ('example.com/blog/' -> 'com,example)/blog'), the prefix keeps it + for boundary in ('/', '?'): + if prefix.endswith(boundary) and not key.endswith(boundary): + key += boundary + return [key], [] if '*' in pattern: raise UsageError(f'invalid URL pattern {pattern!r}, * can only be at the start or the end') return [], [to_surt(pattern)] @@ -52,9 +60,10 @@ def url_hostname(url: str) -> str: def hostname_to_surts(hostname: str) -> List[str]: - """The URLs of this host, and of the host with or without www. (the SURT key drops www.); the exact host is - checked with url_hostname().""" - return [to_surt(normalize_host(hostname) + '/')] + """The URLs of this host, and of the host with or without www. (the SURT key drops www.), with any port; the + exact host is checked with url_hostname().""" + key = to_surt(normalize_host(hostname) + '/') # 'com,example)/' + return [key, key[:-2] + ':'] def domain_to_surts(domain: str) -> List[str]: diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py index a409750..631ceb9 100644 --- a/tests/unit/test_cdx_files.py +++ b/tests/unit/test_cdx_files.py @@ -29,7 +29,10 @@ def count_lines(prefix, day=None): [ ('example.com/*', ['com,example)/'], []), ('example.com/path*', ['com,example)/path'], []), - ('*.example.com', ['com,example)', 'com,example,'], []), + ('example.com/path/*', ['com,example)/path/'], []), + ('example.com/path?*', ['com,example)/path?'], []), + ('example.com:8080/*', ['com,example:8080)/'], []), + ('*.example.com', ['com,example)', 'com,example,', 'com,example:'], []), ('https://www.example.com/a', [], ['com,example)/a']), ], ) @@ -169,8 +172,8 @@ def test_dry_run(capsys): @pytest.mark.parametrize( 'entry, prefixes', [ - ('example.com', ['com,example)', 'com,example,']), - ('https://example.com/', ['com,example)', 'com,example,']), + ('example.com', ['com,example)', 'com,example,', 'com,example:']), + ('https://example.com/', ['com,example)', 'com,example,', 'com,example:']), ('example.com/reports', ['com,example)/reports']), ], ) diff --git a/tests/unit/test_selectors.py b/tests/unit/test_selectors.py index 6b13196..1f3a386 100644 --- a/tests/unit/test_selectors.py +++ b/tests/unit/test_selectors.py @@ -1,22 +1,35 @@ """The URL selectors select the same captures from every backend (CDX files, the columnar index with DuckDB).""" import json +from urllib.parse import urlsplit import pytest from cdx_toolkit.cli import main -from cdx_toolkit.indexes.patterns import to_surt from tests.conftest import requires_duckdb -# url, registered domain +# url, SURT key (as in Common Crawl's indexes, written out instead of computed by the code under test), +# registered domain CAPTURES = [ - ('https://example.com/', 'example.com'), - ('https://www.example.com/w', 'example.com'), - ('https://blog.example.com/', 'example.com'), - ('https://a.blog.example.com/', 'example.com'), - ('https://ablog.example.com/', 'example.com'), - ('https://blogexample.com/', 'blogexample.com'), - ('https://example.com.au/', 'example.com.au'), + ('https://example.com/', 'com,example)/', 'example.com'), + ('https://www.example.com/w', 'com,example)/w', 'example.com'), + ('https://blog.example.com/', 'com,example,blog)/', 'example.com'), + ('https://a.blog.example.com/', 'com,example,blog,a)/', 'example.com'), + ('https://ablog.example.com/', 'com,example,ablog)/', 'example.com'), + ('https://blogexample.com/', 'com,blogexample)/', 'blogexample.com'), + ('https://example.com.au/', 'au,com,example)/', 'example.com.au'), + # path and query boundaries of URL prefixes (the SURT key drops a trailing slash) + ('https://example.org/blog', 'org,example)/blog', 'example.org'), + ('https://example.org/blog/', 'org,example)/blog', 'example.org'), + ('https://example.org/blog/entry', 'org,example)/blog/entry', 'example.org'), + ('https://example.org/blogger', 'org,example)/blogger', 'example.org'), + ('https://example.org/blog?x=1', 'org,example)/blog?x=1', 'example.org'), + # ports, and hosts that only look similar + ('https://example.net:8443/', 'net,example:8443)/', 'example.net'), + ('https://www.example.net/', 'net,example)/', 'example.net'), + ('https://sub.example.net:8443/p', 'net,example,sub:8443)/p', 'example.net'), + ('https://example.net.evil.test/', 'test,evil,net,example)/', 'evil.test'), + ('https://notexample.net/', 'net,notexample)/', 'notexample.net'), ] @@ -27,17 +40,17 @@ def indexes(tmp_path_factory): base = tmp_path_factory.mktemp('indexes') lines = sorted( - f'{to_surt(url)} 20240801120000 ' + f'{key} 20240801120000 ' f'{json.dumps({"url": url, "filename": "a.warc.gz", "offset": str(i), "length": "1"})}\n' - for i, (url, _) in enumerate(CAPTURES) + for i, (url, key, _) in enumerate(CAPTURES) ) (base / 'index.cdx').write_text(''.join(lines)) path = base / 'columnar' / 'crawl=CC-MAIN-2024-33' / 'subset=warc' path.mkdir(parents=True) values = ', '.join( - f"('{to_surt(url)}', '{url}', '{url.split('/')[2]}', '{domain}', '{domain.rsplit('.', 1)[-1]}', {i})" - for i, (url, domain) in enumerate(CAPTURES) + f"('{key}', '{url}', '{urlsplit(url).hostname}', '{domain}', '{domain.rsplit('.', 1)[-1]}', {i})" + for i, (url, key, domain) in enumerate(CAPTURES) ) duckdb.connect().execute(f''' COPY (SELECT *, array_to_string(list_reverse(string_split(url_host_name, '.')), '.') AS url_host_name_reversed, @@ -77,6 +90,24 @@ def indexes(tmp_path_factory): ['https://example.com.au/', 'https://www.example.com/w']), (['--url', '*.blog.example.com'], ['https://a.blog.example.com/', 'https://blog.example.com/']), (['--url', 'blog.example.com/*'], ['https://blog.example.com/']), + # URL prefixes keep a trailing / or ? as a boundary (like the CDX API) + (['--url', 'example.org/blog/*'], ['https://example.org/blog/entry']), + (['--url', 'example.org/blog*'], [ + 'https://example.org/blog', 'https://example.org/blog/', 'https://example.org/blog/entry', + 'https://example.org/blog?x=1', 'https://example.org/blogger', + ]), + (['--url', 'example.org/blog?*'], ['https://example.org/blog?x=1']), + (['--url', 'example.org/blog/?*'], ['https://example.org/blog?x=1']), + # hosts with a port + (['--hostname', 'example.net'], ['https://example.net:8443/']), + (['--hostname', 'sub.example.net'], ['https://sub.example.net:8443/p']), + (['--domain', 'example.net'], [ + 'https://example.net:8443/', 'https://sub.example.net:8443/p', 'https://www.example.net/', + ]), + (['--url', '*.example.net'], [ + 'https://example.net:8443/', 'https://sub.example.net:8443/p', 'https://www.example.net/', + ]), + (['--url', 'example.net:8443/*'], ['https://example.net:8443/']), ]) def test_selectors_match_the_same_captures_on_every_backend(indexes, capsys, selector, expected): for index in indexes: diff --git a/tests/unit/test_sql_builder.py b/tests/unit/test_sql_builder.py index 7020b99..811456a 100644 --- a/tests/unit/test_sql_builder.py +++ b/tests/unit/test_sql_builder.py @@ -107,4 +107,4 @@ def test_build_query_ip_addresses_without_reversed_host(): def test_build_query_url_with_host_range(): q = build_sql(IndexQuery(urls=['www.example.com/blog/*']), 'ccindex') assert ("(url_host_name_reversed >= 'com.example' AND url_host_name_reversed < 'com.example/' " - "AND url_surtkey LIKE 'com,example)/blog%' ESCAPE '\\')") in q + "AND url_surtkey LIKE 'com,example)/blog/%' ESCAPE '\\')") in q diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index f9135ea..135945c 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -137,7 +137,8 @@ def test_guided_query_url_patterns(): f"OR ({range_org} AND url_surtkey = 'org,example)/') " f"OR ({range_com} AND url_surtkey LIKE 'com,example)/a\\_b%' ESCAPE '\\') " f"OR ({range_net} AND url_surtkey LIKE 'net,example)%' ESCAPE '\\') " - f"OR ({range_net} AND url_surtkey LIKE 'net,example,%' ESCAPE '\\'))") in sql + f"OR ({range_net} AND url_surtkey LIKE 'net,example,%' ESCAPE '\\') " + f"OR ({range_net} AND url_surtkey LIKE 'net,example:%' ESCAPE '\\'))") in sql def test_guided_query_time_range_and_languages(): From be68d860bc9ca24c0dd68fb2a5c0bbd9553a57d6 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 17:50:37 +0200 Subject: [PATCH 71/91] fix(cdx-api): --hostname selects exactly the host matchType=host selects the SURT host, the same for example.com and www.example.com; the captures of the other host are now dropped before --limit, which is then not sent to the server. --count stays an estimate of the SURT host (documented). --- README.md | 6 ++-- cdx_toolkit/indexes/backends/cdx_api.py | 29 +++++++++++++----- tests/unit/test_indexes.py | 40 +++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7797fc9..5e45665 100644 --- a/README.md +++ b/README.md @@ -151,8 +151,10 @@ extension but use the CDXJ format (crawl index JSON). Use `--fields url,status,t fields are written. `--count` prints the number of captures instead. For the CDX API this is -a cheap estimate based on the number of index pages; for CDX files it is an -exact count, and for SQL indexes an exact `count(*)`. +a cheap estimate based on the number of index pages (for `--hostname`, of the +host with and without `www.`, which the CDX API does not distinguish; the +captures of the other one are dropped when they are read); for CDX files it is +an exact count, and for SQL indexes an exact `count(*)`. If you don't specify the crawls or dates you're interested in, queries of `cc` are limited to the CC indexes of the most recent year, to diff --git a/cdx_toolkit/indexes/backends/cdx_api.py b/cdx_toolkit/indexes/backends/cdx_api.py index 1d7a622..9f66bed 100644 --- a/cdx_toolkit/indexes/backends/cdx_api.py +++ b/cdx_toolkit/indexes/backends/cdx_api.py @@ -7,6 +7,7 @@ from cdx_toolkit.compat import munge_filter from ..base import IndexBackend, QueryPlan, UsageError +from ..patterns import normalize_host, url_hostname from ..query import IndexQuery @@ -44,7 +45,10 @@ def validate(self, query: IndexQuery) -> None: ) from None def plan(self, query: IndexQuery) -> 'ApiPlan': - requests = [ApiRequest(url, api_params(query, match_type)) for url, match_type in query.url_patterns()] + requests = [] + for url, match_type in query.url_patterns(): + hostname = normalize_host(url) if match_type == 'host' else None + requests.append(ApiRequest(hostname or url, api_params(query, match_type), hostname=hostname)) return ApiPlan(self.source, requests, query.limit, query.crawl) def select(self, query: IndexQuery) -> Iterator[Mapping]: @@ -57,6 +61,8 @@ def select(self, query: IndexQuery) -> Iterator[Mapping]: # closest only makes sense for a single (non-paged) query captures = cdx.get(request.url, **params) if query.closest else cdx.iter(request.url, **params) for capture in captures: + if request.hostname and url_hostname(capture.get('url', '')) != request.hostname: + continue if seen is not None: identity = capture_identity(capture) if identity in seen: @@ -66,7 +72,7 @@ def select(self, query: IndexQuery) -> Iterator[Mapping]: def select_count(self, query: IndexQuery) -> int: """Estimate from the number of index pages, at most --limit; overlapping selectors are counted more - than once.""" + than once, and --hostname counts the host with and without www. (the server's matchType=host).""" cdx = self._fetcher(query) count = sum( cdx.get_size_estimate(request.url, **request.params_with(query.limit)) @@ -87,10 +93,14 @@ def _fetcher(self, query: IndexQuery) -> cdx_toolkit.CDXFetcher: @dataclass class ApiRequest: - """One request (series of paged requests) to the CDX API: a URL and its parameters (without the limit).""" + """One request (series of paged requests) to the CDX API: a URL and its parameters (without the limit). + + matchType=host selects the SURT host, which is the same for example.com and www.example.com: with `hostname`, + only the captures of exactly this host are kept.""" url: str params: dict + hostname: Optional[str] = None def params_with(self, limit: Optional[int]) -> dict: return {'limit': limit, **self.params} if limit else dict(self.params) @@ -107,15 +117,20 @@ class ApiPlan(QueryPlan): @property def server_limit(self) -> Optional[int]: - """The limit sent to the server: only for a single request; with several, the captures selected by more - than one request are removed first (and the limit is applied by IndexBackend.records()).""" - return self.limit if len(self.requests) == 1 else None + """The limit sent to the server: only for a single request whose captures are all kept; with several, the + captures selected by more than one request are removed first, and with --hostname, the captures of the + other host (with or without www.), and the limit is applied by IndexBackend.records().""" + return self.limit if len(self.requests) == 1 and not self.requests[0].hostname else None def describe(self) -> str: lines = [] for request in self.requests: - params = ' '.join(f'{k}={v}' for k, v in request.params_with(self.limit).items()) + params = ' '.join(f'{k}={v}' for k, v in request.params_with(self.server_limit).items()) lines.append(f'{self.source}: url={request.url} {params}'.rstrip()) + if request.hostname: + lines.append(f' only captures of the host {request.hostname}') + if self.limit and self.server_limit is None: + lines.append(f'limit: {self.limit} (after removing duplicates and other hosts)') if self.crawl: lines.append(f'crawls: {self.crawl}') return '\n'.join(lines) + '\n' diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py index 034f2e0..5e11413 100644 --- a/tests/unit/test_indexes.py +++ b/tests/unit/test_indexes.py @@ -195,3 +195,43 @@ def test_records_url_fgrep(): records = list(backend.records(query)) assert [r['url'] for r in records] == ['http://example.com/keep/this'] + + +def test_cdx_api_hostname_is_exact(): + """matchType=host selects the host with and without www.: the other host's captures are dropped before the + limit (which is not sent to the server).""" + results = { + 'www.example.com': [ + _capture('http://example.com/'), _capture('https://EXAMPLE.com/a'), + _capture('http://www.example.com/'), _capture('https://www.example.com:8443/b'), + _capture('http://www.example.com/c'), + ], + 'example.org/*': [_capture('http://example.org/')], + } + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.iter.side_effect = lambda url, **params: iter(results[url]) + + records = list(CdxApiBackend('cc').records(IndexQuery(hostnames=['WWW.example.com.'], limit=2))) + assert [r['url'] for r in records] == ['http://www.example.com/', 'https://www.example.com:8443/b'] + assert fetcher.return_value.iter.call_args.kwargs == {'matchType': 'host'} # no server limit + + # combined with another selector: a union + query = IndexQuery(hostnames=['www.example.com'], urls=['example.org/*']) + records = list(CdxApiBackend('cc').records(query)) + assert [r['url'] for r in records] == [ + 'http://example.org/', 'http://www.example.com/', 'https://www.example.com:8443/b', + 'http://www.example.com/c', + ] + + plan = CdxApiBackend('cc').explain(IndexQuery(hostnames=['www.example.com'], limit=2)) + assert plan == ('cc: url=www.example.com matchType=host\n only captures of the host www.example.com\n' + 'limit: 2 (after removing duplicates and other hosts)\n') + + +def test_cdx_api_closest_with_hostname(): + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.get.return_value = [_capture('http://example.com/'), _capture('http://www.example.com/')] + query = IndexQuery(hostnames=['www.example.com'], closest='20170615', limit=1) + records = list(CdxApiBackend('cc').records(query)) + assert [r['url'] for r in records] == ['http://www.example.com/'] + assert fetcher.return_value.get.call_args.kwargs == {'closest': '20170615', 'matchType': 'host'} From 7fae406bc66ed0b33df4bdb988f37c9ff74e5d26 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 17:54:17 +0200 Subject: [PATCH 72/91] fix(repackage): a blocked source no longer delays the writer's error or Ctrl-C The source ran in asyncio.to_thread, whose default executor asyncio.run() waits for at shutdown: with stdin from a process that keeps running (or a blocking generator), a failed writer or Ctrl-C hung. The source now runs in a daemon thread that nothing waits for (run_in_daemon_thread, as for the HTTP reads), and stops before its next record once the pipeline stops. --- cdx_toolkit/repackage/repackager.py | 16 +++- tests/repackage/test_repackager.py | 119 ++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 4 deletions(-) diff --git a/cdx_toolkit/repackage/repackager.py b/cdx_toolkit/repackage/repackager.py index 7bee60d..1e5fa84 100644 --- a/cdx_toolkit/repackage/repackager.py +++ b/cdx_toolkit/repackage/repackager.py @@ -12,6 +12,7 @@ from cdx_toolkit.aws import botocore_config from cdx_toolkit.repackage.jobs import RangePayload, RepackageResult from cdx_toolkit.repackage.readers import Readers +from cdx_toolkit.repackage.readers.http import run_in_daemon_thread from cdx_toolkit.repackage.urls import is_s3_url from cdx_toolkit.repackage.writers.series import WarcFileSeries @@ -196,16 +197,23 @@ async def run_async(self) -> RepackageResult: ) async def _produce_range_jobs(self, range_jobs_queue: asyncio.Queue, stop: threading.Event) -> int: - """Drive the (sync) source in a worker thread, feeding the async queue. + """Drive the (sync) source in a daemon thread, feeding the async queue. The thread waits for queue space in short steps, so it stops soon after `stop` is set (cancellation, - or a failed writer), also when the queue is full and nobody reads from it anymore.""" + or a failed writer), also when the queue is full and nobody reads from it anymore. A source that blocks + (e.g. stdin from a process that is still running) cannot be interrupted: nothing waits for its thread, + not this task when it is cancelled, not asyncio.run() and not the interpreter at exit.""" loop = asyncio.get_running_loop() def drain() -> int: count = 0 for job in self.source.iter_range_jobs(): - future = asyncio.run_coroutine_threadsafe(range_jobs_queue.put(job), loop) + if stop.is_set(): + return count + try: + future = asyncio.run_coroutine_threadsafe(range_jobs_queue.put(job), loop) + except RuntimeError: # the event loop is closed + return count while True: if stop.is_set(): future.cancel() @@ -219,7 +227,7 @@ def drain() -> int: return count try: - count = await asyncio.to_thread(drain) + count = await run_in_daemon_thread(drain) finally: stop.set() # the thread keeps running if this task is cancelled; let it stop diff --git a/tests/repackage/test_repackager.py b/tests/repackage/test_repackager.py index 46e7358..34fe93f 100644 --- a/tests/repackage/test_repackager.py +++ b/tests/repackage/test_repackager.py @@ -7,6 +7,7 @@ import itertools import logging import os +import sys import threading import time from unittest.mock import Mock, patch @@ -560,3 +561,121 @@ def test_log_progress(caplog): assert caplog.text.count('WARC Reader 1: ') == 2 assert 'WARC Reader 1: 4 items, 4.0 MB in ' in caplog.text + + +def test_writer_error_with_a_blocked_source(tmp_path): + """The writer fails while the source waits for its next capture (e.g. a generator waiting for input): run() + raises the writer's error at once, without waiting for the source.""" + release = threading.Event() + + def captures(): + yield CAPTURES[0] + release.wait() # blocks until the end of the test + yield CAPTURES[1] + + async def failing_write(self, data): + raise OSError('disk full') + + outcome = [] + + def main(): + repackager = Repackager( + source=RecordsSource(captures(), 'https://data.example.org', sort=False), + prefix_path=str(tmp_path / 'out'), writer_info={}, + ) + try: + repackager.run() + except OSError as e: + outcome.append(e) + + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ + patch.object(LocalOutputFile, 'write', failing_write): + thread = threading.Thread(target=main, daemon=True) + thread.start() + thread.join(timeout=5) + release.set() + assert not thread.is_alive(), 'run() waits for the blocked source' + assert [str(e) for e in outcome] == ['disk full'] + assert glob.glob(str(tmp_path / 'out-*')) == [] + + +CHILD = ''' +import sys +from unittest.mock import patch +from cdx_toolkit.cli import main +from cdx_toolkit.repackage.writers.local import LocalOutputFile +from tests.repackage.test_repackager import fake_get + +async def failing_write(self, data): + raise OSError('disk full') + +with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \\ + patch.object(LocalOutputFile, 'write', failing_write): + main(['repackage', '-', '--no-sort', '--warc-download-prefix', 'https://data.example.org', + '--prefix', sys.argv[1]]) +''' + + +def test_writer_error_with_stdin_still_open(tmp_path): + """cdxt repackage reading stdin from a process that keeps running: the writer's error ends cdxt at once.""" + import json + import subprocess + import sys + + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + proc = subprocess.Popen( + [sys.executable, '-c', CHILD, str(tmp_path / 'out')], cwd=root, stdin=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, + ) + try: + proc.stdin.write(json.dumps(CAPTURES[0]) + '\n') + proc.stdin.flush() # and stdin stays open + proc.wait(timeout=30) + stderr = proc.stderr.read() + finally: + if proc.poll() is None: + proc.kill() + proc.stdin.close() + assert proc.returncode != 0 + assert 'disk full' in stderr + assert glob.glob(str(tmp_path / 'out-*')) == [] + + +CHILD_WITHOUT_FAILURE = ''' +import sys +from unittest.mock import patch +from cdx_toolkit.cli import main +from tests.repackage.test_repackager import fake_get + +with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): + main(['repackage', '-', '--no-sort', '--warc-download-prefix', 'https://data.example.org', + '--prefix', sys.argv[1]]) +''' + + +@pytest.mark.skipif(sys.platform == 'win32', reason='SIGINT') +def test_ctrl_c_with_stdin_still_open(tmp_path): + """Ctrl-C while cdxt repackage waits for more input on stdin: it ends at once, and removes the current file.""" + import json + import signal + import subprocess + + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + proc = subprocess.Popen( + [sys.executable, '-c', CHILD_WITHOUT_FAILURE, str(tmp_path / 'out')], cwd=root, stdin=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, + ) + try: + proc.stdin.write(json.dumps(CAPTURES[0]) + '\n') + proc.stdin.flush() # and stdin stays open + deadline = time.monotonic() + 30 + while not glob.glob(str(tmp_path / 'out-*')) and time.monotonic() < deadline: + time.sleep(0.05) # the first record is being written + proc.send_signal(signal.SIGINT) + proc.wait(timeout=30) + finally: + if proc.poll() is None: + proc.kill() + proc.stdin.close() + assert proc.returncode != 0 + assert glob.glob(str(tmp_path / 'out-*')) == [] From 38974bb96f45327ac68a69cce90b894caff8827d Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 17:57:59 +0200 Subject: [PATCH 73/91] fix(repackage): a failing process stops the other processes at once ex.map() returned the results in order and the executor waited for all workers, so the other processes ran their shards to the end. The workers now share a cancel event that each pipeline watches (Repackager(cancel=...) aborts the current file and raises RepackageCancelled); the parent raises the first failure (or Ctrl-C) as soon as it happens, after setting the event, cancelling the pending workers and terminating those that have not stopped after 60 s. --- README.md | 8 ++- cdx_toolkit/repackage/multiprocess.py | 64 ++++++++++++++++++--- cdx_toolkit/repackage/repackager.py | 28 ++++++++- tests/conftest.py | 13 +++++ tests/repackage/test_multiprocess_cancel.py | 55 ++++++++++++++++++ tests/repackage/test_multiprocess_naming.py | 7 +-- tests/repackage/test_repackager.py | 37 ++++++++++++ 7 files changed, 194 insertions(+), 18 deletions(-) create mode 100644 tests/repackage/test_multiprocess_cancel.py diff --git a/README.md b/README.md index 5e45665..36e2a2d 100644 --- a/README.md +++ b/README.md @@ -305,9 +305,11 @@ in the `warcinfo` record, and `--add-metadata-record FILE` to add a file A capture that cannot be fetched (after retries) does not stop the job: the other captures are still written, and at the end `cdxt repackage` logs how many captures were requested, written and failed, and exits with status 1. -A failure to write the output stops the job at once; the WARC file being -written is removed (or its S3 upload aborted), the files before it are -complete. +A failure to write the output stops the job at once, with `--processes` all +processes; the WARC files being written are removed (or their S3 uploads +aborted), the files before them are complete. A process that has not stopped +after 60 seconds is terminated, which can leave an incomplete S3 multipart +upload (an S3 lifecycle rule for incomplete uploads removes it). ### Migrating from the old commands diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index 5f23f7c..e0124a8 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -16,6 +16,9 @@ record names the file it is in). Which worker writes which number depends on timing. Output honours ``--size`` as single-process mode does (files are a target size; the last file of each worker may be short, so up to N short files appear). +4. A failing worker (or Ctrl-C) stops all of them: the parent sets an event shared by all workers, whose + pipelines then abort the file they are writing (completed files are kept). A worker that has not + stopped after ``CANCEL_TIMEOUT`` seconds is terminated. The WARC writer supports local and S3 output. For other fsspec destinations (e.g. an ``hf://`` bucket) the files are staged in a local directory under their final names and @@ -32,8 +35,8 @@ import time import fsspec -from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor -from typing import Dict, List, Optional, Tuple +from concurrent.futures import FIRST_EXCEPTION, ProcessPoolExecutor, ThreadPoolExecutor, wait +from typing import Callable, Dict, List, Optional, Tuple from cdx_toolkit.aws import configure_fsspec_s3 from cdx_toolkit.record_io import RecordWriter, read_records @@ -47,8 +50,12 @@ # Concurrency for copying staged files to a non-S3 remote destination (override with CDXT_COPY_CONCURRENCY). _COPY_CONCURRENCY = int(os.environ.get('CDXT_COPY_CONCURRENCY', '16')) -# The shared counter of output file numbers, set in each worker process by _init_worker. +# Seconds the workers get to stop (and abort their current file) after a failure, before they are terminated. +CANCEL_TIMEOUT = 60 + +# The shared counter of output file numbers and the cancel event, set in each worker process by _init_worker. _FILE_SEQUENCE = None +_CANCEL = None def _output_filename(prefix_path: str, sequence: int, subprefix: Optional[str] = None, @@ -61,9 +68,10 @@ def _output_filename(prefix_path: str, sequence: int, subprefix: Optional[str] = return generate_warc_filename(prefix_path, sequence=sequence, writer_subprefix=subprefix, gzip=gzip) -def _init_worker(file_sequence) -> None: - global _FILE_SEQUENCE +def _init_worker(file_sequence, cancel) -> None: + global _FILE_SEQUENCE, _CANCEL _FILE_SEQUENCE = file_sequence + _CANCEL = cancel configure_fsspec_s3() # a new process does not have the fsspec configuration of the main process @@ -96,10 +104,51 @@ def _run_worker(cfg: Dict) -> RepackageResult: max_attempts=cfg['max_attempts'], max_file_size=cfg['max_file_size'], # rotate at --size next_file_sequence=_next_file_sequence, + cancel=_CANCEL, ) return repackager.run() +def run_worker_processes(worker: Callable, configs: List, file_sequence, mp_context, + cancel_timeout: float = None) -> List: + """Run worker(config) for each config, each in its own process; the results in the order of the configs. + + The first failure (or KeyboardInterrupt) is raised as soon as it happens, after the other workers were + stopped: the shared cancel event is set, workers that have not started are cancelled, and running workers + get `cancel_timeout` seconds to stop (abort their current file) before they are terminated. A terminated + worker cannot abort an S3 multipart upload; an S3 lifecycle rule removes incomplete uploads.""" + cancel_timeout = CANCEL_TIMEOUT if cancel_timeout is None else cancel_timeout + cancel = mp_context.Event() + ex = ProcessPoolExecutor( + max_workers=len(configs), mp_context=mp_context, initializer=_init_worker, initargs=(file_sequence, cancel) + ) + futures = [] + try: + futures = [ex.submit(worker, config) for config in configs] + done, _ = wait(futures, return_when=FIRST_EXCEPTION) + for future in done: + future.result() # raises the first failure + results = [future.result() for future in futures] + except BaseException: + cancel.set() + for future in futures: + future.cancel() + _, running = wait(futures, timeout=cancel_timeout) + if running: + logger.error('%d worker(s) did not stop within %ds, terminating them (an S3 multipart upload of an ' + 'unfinished file may be left incomplete)', len(running), cancel_timeout) + # ProcessPoolExecutor.terminate_workers() is new in Python 3.14 + processes = list((getattr(ex, '_processes', None) or {}).values()) + for process in processes: + process.terminate() + for process in processes: + process.join(timeout=10) + ex.shutdown(wait=False, cancel_futures=True) + raise + ex.shutdown(wait=True) + return results + + def _shard_record(job) -> dict: """The record of a job in a shard: the WARC location with the full WARC URL as filename (read without a download prefix), or the capture of a wayback replay.""" @@ -185,10 +234,7 @@ def run_multiprocess_repackage( logger.info('Launching %d worker processes x %d readers', len(configs), readers_per_process) mp_context = multiprocessing.get_context() file_sequence = mp_context.Value('i', 0) - with ProcessPoolExecutor( - max_workers=len(configs), mp_context=mp_context, initializer=_init_worker, initargs=(file_sequence,) - ) as ex: - results = list(ex.map(_run_worker, configs)) + results = run_worker_processes(_run_worker, configs, file_sequence, mp_context) result = sum(results, RepackageResult()) total = result.written n_files = file_sequence.value diff --git a/cdx_toolkit/repackage/repackager.py b/cdx_toolkit/repackage/repackager.py index 1e5fa84..8ad15cf 100644 --- a/cdx_toolkit/repackage/repackager.py +++ b/cdx_toolkit/repackage/repackager.py @@ -58,6 +58,10 @@ def summary(self) -> str: ) +class RepackageCancelled(Exception): + """The pipeline was stopped from outside (its `cancel` event was set), e.g. because another process failed.""" + + class Repackager: """Copy WARC records, byte for byte, from the WARC files of the range jobs (or the records replayed from a wayback machine) into a series of WARC files. @@ -89,6 +93,7 @@ def __init__( min_part_size: int = 5 * 1024 * 1024, # 5 MiB (for upload) max_file_size: Optional[int] = 1 * 1024 * 1024 * 1024, # 1 GiB (for WARC outputs) next_file_sequence: Optional[Callable[[int], int]] = None, + cancel=None, ): """Set up the pipeline. @@ -110,6 +115,8 @@ def __init__( max_file_size: Maximum byte size for individual WARC files (default: 1 GiB). next_file_sequence: Number of the next output file, given the current one (default: current + 1). + cancel: Optional event (threading.Event, multiprocessing.Event): once it is set, the pipeline stops + like on a failure (the current file is aborted) and run() raises RepackageCancelled. """ if n_parallel_readers < 1: raise ValueError(f'n_parallel_readers must be at least 1, not {n_parallel_readers}') @@ -138,6 +145,7 @@ def __init__( self.min_part_size = min_part_size self.max_file_size = max_file_size self.next_file_sequence = next_file_sequence # see WarcFileSeries + self.cancel = cancel def run(self) -> RepackageResult: """Run the pipeline (runs run_async in an event loop). @@ -154,7 +162,19 @@ def run(self) -> RepackageResult: logger.info('Using uvloop event loop') except ImportError: logger.warning('CDXT_UVLOOP=1 but uvloop is not installed; using default asyncio loop') - return runner(self.run_async()) + try: + return runner(self.run_async()) + except asyncio.CancelledError: + if self.cancel is not None and self.cancel.is_set(): + raise RepackageCancelled('stopped (cancel event)') from None + raise + + async def _watch_cancel(self, pipeline: asyncio.Task) -> None: + """Cancel the pipeline once the cancel event is set.""" + while not self.cancel.is_set(): + await asyncio.sleep(0.1) + logger.warning('Stopping: cancelled from outside') + pipeline.cancel() def s3_read_config(self): """botocore config of the S3 client for range reads: high throughput.""" @@ -290,12 +310,18 @@ async def feed(): feeder = asyncio.create_task(feed()) tasks = [producer, *warc_readers, warc_writer, feeder] + watcher = None + if self.cancel is not None: + watcher = asyncio.create_task(self._watch_cancel(asyncio.current_task())) + tasks.append(watcher) try: done, _ = await asyncio.wait({feeder, warc_writer}, return_when=asyncio.FIRST_EXCEPTION) for task in done: task.result() # raises the exception of a failed task requested, readers_results = await feeder writer_tracker = await warc_writer + if watcher is not None: + watcher.cancel() except BaseException: stop.set() for task in tasks: diff --git a/tests/conftest.py b/tests/conftest.py index 195b5b4..b124016 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -363,6 +363,19 @@ def __exit__(self, *exc): def map(self, fn, items): return [fn(item) for item in items] + def submit(self, fn, *args): + import concurrent.futures + + future = concurrent.futures.Future() + try: + future.set_result(fn(*args)) + except Exception as e: + future.set_exception(e) + return future + + def shutdown(self, wait=True, cancel_futures=False): + pass + class S3Env: """S3 for a test: a local moto server with the buckets commoncrawl (the WARC files of WARC_RECORDS) and diff --git a/tests/repackage/test_multiprocess_cancel.py b/tests/repackage/test_multiprocess_cancel.py new file mode 100644 index 0000000..95edfc4 --- /dev/null +++ b/tests/repackage/test_multiprocess_cancel.py @@ -0,0 +1,55 @@ +"""A failing worker process stops the others (real spawned processes).""" +import multiprocessing +import os +import time + +import pytest + +from cdx_toolkit.repackage import multiprocess +from cdx_toolkit.repackage.multiprocess import run_worker_processes + + +def worker(config): + """'fail': fails after a moment; 'wait': waits for the cancel event, then stops (or completes after 20 s); + 'ignore': does not look at the cancel event.""" + kind, marker = config + if kind == 'fail': + time.sleep(0.2) + raise OSError('disk full') + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + if kind == 'wait' and multiprocess._CANCEL.is_set(): + return 'stopped' + time.sleep(0.05) + with open(marker, 'w') as f: + f.write('completed') + return 'completed' + + +@pytest.mark.parametrize('failing_first', [True, False]) +def test_a_failing_worker_stops_the_others(tmp_path, failing_first): + configs = [('wait', str(tmp_path / 'a')), ('wait', str(tmp_path / 'b'))] + configs.insert(0 if failing_first else len(configs), ('fail', None)) + ctx = multiprocessing.get_context('spawn') + start = time.monotonic() + with pytest.raises(OSError, match='disk full'): + run_worker_processes(worker, configs, ctx.Value('i', 0), ctx, cancel_timeout=15) + assert time.monotonic() - start < 15 # the others did not run to completion (20 s) + assert os.listdir(tmp_path) == [] + + +def test_a_worker_that_does_not_stop_is_terminated(tmp_path): + configs = [('fail', None), ('ignore', str(tmp_path / 'a'))] + ctx = multiprocessing.get_context('spawn') + start = time.monotonic() + with pytest.raises(OSError, match='disk full'): + run_worker_processes(worker, configs, ctx.Value('i', 0), ctx, cancel_timeout=1) + assert time.monotonic() - start < 15 + time.sleep(0.5) + assert os.listdir(tmp_path) == [] + + +def test_results_in_the_order_of_the_configs(tmp_path): + ctx = multiprocessing.get_context('spawn') + file_sequence = ctx.Value('i', 0) + assert run_worker_processes(abs, [-3, 1, -2], file_sequence, ctx) == [3, 1, 2] diff --git a/tests/repackage/test_multiprocess_naming.py b/tests/repackage/test_multiprocess_naming.py index 81ab8ae..a7f61aa 100644 --- a/tests/repackage/test_multiprocess_naming.py +++ b/tests/repackage/test_multiprocess_naming.py @@ -5,7 +5,6 @@ """ import multiprocessing import os -from concurrent.futures import ProcessPoolExecutor from cdx_toolkit.repackage import multiprocess from cdx_toolkit.repackage.multiprocess import _copy_staged_files, _output_filename @@ -29,10 +28,8 @@ def _take_numbers(n): def test_file_numbers_are_shared_across_processes(): ctx = multiprocessing.get_context() counter = ctx.Value('i', 0) - with ProcessPoolExecutor( - max_workers=3, mp_context=ctx, initializer=multiprocess._init_worker, initargs=(counter,) - ) as ex: - numbers = [n for result in ex.map(_take_numbers, [5, 5, 5]) for n in result] + results = multiprocess.run_worker_processes(_take_numbers, [5, 5, 5], counter, ctx) + numbers = [n for result in results for n in result] # every file number is used exactly once, without gaps assert sorted(numbers) == list(range(1, 16)) diff --git a/tests/repackage/test_repackager.py b/tests/repackage/test_repackager.py index 34fe93f..c981ddb 100644 --- a/tests/repackage/test_repackager.py +++ b/tests/repackage/test_repackager.py @@ -679,3 +679,40 @@ def test_ctrl_c_with_stdin_still_open(tmp_path): proc.stdin.close() assert proc.returncode != 0 assert glob.glob(str(tmp_path / 'out-*')) == [] + + +def test_cancel_event_stops_the_pipeline_and_removes_the_file(tmp_path): + """The event set by the parent of the worker processes (when another worker failed).""" + from cdx_toolkit.repackage.repackager import RepackageCancelled + + release, cancel = threading.Event(), threading.Event() + + def captures(): + yield CAPTURES[0] + release.wait() + yield CAPTURES[1] + + outcome = [] + + def main(): + repackager = Repackager( + source=RecordsSource(captures(), 'https://data.example.org', sort=False), + prefix_path=str(tmp_path / 'out'), writer_info={}, cancel=cancel, + ) + try: + outcome.append(repackager.run()) + except RepackageCancelled as e: + outcome.append(e) + + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): + thread = threading.Thread(target=main, daemon=True) + thread.start() + deadline = time.monotonic() + 10 + while not glob.glob(str(tmp_path / 'out-*')) and time.monotonic() < deadline: + time.sleep(0.01) # the first record is being written + cancel.set() + thread.join(timeout=5) + release.set() + assert not thread.is_alive() + assert len(outcome) == 1 and isinstance(outcome[0], RepackageCancelled) + assert glob.glob(str(tmp_path / 'out-*')) == [] From 42db6fc7d1f84f0659568a32d4e055921b46258f Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 17:59:21 +0200 Subject: [PATCH 74/91] fix: the worker processes of cdxt index (one output per CDX file) use the S3 defaults too A spawned process (macOS, Windows) does not inherit the fsspec configuration set by the main process; only the repackage workers configured it. Both process pools now come from aws.process_pool(), which applies configure_fsspec_s3() in every new process. --- cdx_toolkit/aws.py | 18 +++++++++++++++ cdx_toolkit/indexes/backends/cdx_files.py | 4 ++-- cdx_toolkit/repackage/multiprocess.py | 9 +++----- tests/repackage/test_repackage_cli.py | 2 +- tests/repackage/test_repackager.py | 8 +++---- tests/unit/test_aws.py | 28 +++++++++++++++++++++++ 6 files changed, 56 insertions(+), 13 deletions(-) diff --git a/cdx_toolkit/aws.py b/cdx_toolkit/aws.py index 9d37c43..1bb4797 100644 --- a/cdx_toolkit/aws.py +++ b/cdx_toolkit/aws.py @@ -149,6 +149,24 @@ def configure_fsspec_s3() -> None: conf.setdefault('max_concurrency', max_concurrent_requests()) +def process_pool(max_workers: int, mp_context=None, initializer=None, initargs=()): + """A ProcessPoolExecutor whose processes use the S3 defaults of configure_fsspec_s3() like this one: a process + started with spawn (macOS, Windows) does not inherit the changes of fsspec.config.conf. Settings from + files and environment variables apply in every process; other in-memory changes of the configuration + are not passed on.""" + from concurrent.futures import ProcessPoolExecutor + + return ProcessPoolExecutor( + max_workers=max_workers, mp_context=mp_context, initializer=_init_process, initargs=(initializer, initargs) + ) + + +def _init_process(initializer, initargs) -> None: + configure_fsspec_s3() + if initializer is not None: + initializer(*initargs) + + def _user_value(setting: str) -> Optional[str]: for name in SETTING_ENV_VARS.get(setting, ()): if os.environ.get(name): diff --git a/cdx_toolkit/indexes/backends/cdx_files.py b/cdx_toolkit/indexes/backends/cdx_files.py index 8f9dd91..cba0ebd 100644 --- a/cdx_toolkit/indexes/backends/cdx_files.py +++ b/cdx_toolkit/indexes/backends/cdx_files.py @@ -7,11 +7,11 @@ ''' import logging import os -from concurrent.futures import ProcessPoolExecutor from dataclasses import dataclass from itertools import islice from typing import Iterator, List, Mapping, NamedTuple, Optional, Tuple +from cdx_toolkit.aws import process_pool from cdx_toolkit.record_io import ( check_glob, format_from_path, is_empty_dir, is_glob, open_text, parse_cdxj_line, read_records, ) @@ -263,7 +263,7 @@ def write_per_file(self, query: IndexQuery, output_base: str, processes: int = 1 tasks = [(f.path, join(f.relative_path), capture_filter) for f in self.files()] if processes > 1: - with ProcessPoolExecutor(max_workers=processes) as ex: + with process_pool(processes) as ex: results = list(ex.map(_filter_file, tasks)) else: results = [_filter_file(task) for task in tasks] diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index e0124a8..3c9c48d 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -35,10 +35,10 @@ import time import fsspec -from concurrent.futures import FIRST_EXCEPTION, ProcessPoolExecutor, ThreadPoolExecutor, wait +from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait from typing import Callable, Dict, List, Optional, Tuple -from cdx_toolkit.aws import configure_fsspec_s3 +from cdx_toolkit.aws import process_pool from cdx_toolkit.record_io import RecordWriter, read_records from cdx_toolkit.repackage.jobs import RecordsSource, RepackageResult, WaybackJob, WaybackSource from cdx_toolkit.repackage.repackager import Repackager @@ -72,7 +72,6 @@ def _init_worker(file_sequence, cancel) -> None: global _FILE_SEQUENCE, _CANCEL _FILE_SEQUENCE = file_sequence _CANCEL = cancel - configure_fsspec_s3() # a new process does not have the fsspec configuration of the main process def _next_file_sequence(_current: int) -> int: @@ -119,9 +118,7 @@ def run_worker_processes(worker: Callable, configs: List, file_sequence, mp_cont worker cannot abort an S3 multipart upload; an S3 lifecycle rule removes incomplete uploads.""" cancel_timeout = CANCEL_TIMEOUT if cancel_timeout is None else cancel_timeout cancel = mp_context.Event() - ex = ProcessPoolExecutor( - max_workers=len(configs), mp_context=mp_context, initializer=_init_worker, initargs=(file_sequence, cancel) - ) + ex = process_pool(len(configs), mp_context=mp_context, initializer=_init_worker, initargs=(file_sequence, cancel)) futures = [] try: futures = [ex.submit(worker, config) for config in configs] diff --git a/tests/repackage/test_repackage_cli.py b/tests/repackage/test_repackage_cli.py index 34c0424..e8924f2 100644 --- a/tests/repackage/test_repackage_cli.py +++ b/tests/repackage/test_repackage_cli.py @@ -121,7 +121,7 @@ def test_repackage_over_http_with_parallel_readers(tmpdir): def test_repackage_over_http_with_processes(tmpdir, monkeypatch, mock_data_commoncrawl): if mock_data_commoncrawl is not None: # the HTTP mock only applies in this process - monkeypatch.setattr('cdx_toolkit.repackage.multiprocess.ProcessPoolExecutor', InlineExecutor) + monkeypatch.setattr('cdx_toolkit.repackage.multiprocess.process_pool', InlineExecutor) assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=tmpdir, extra_args=['--processes=2']) diff --git a/tests/repackage/test_repackager.py b/tests/repackage/test_repackager.py index c981ddb..17b3372 100644 --- a/tests/repackage/test_repackager.py +++ b/tests/repackage/test_repackager.py @@ -363,7 +363,7 @@ def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): with patch('aioboto3.Session') as session, \ patch('cdx_toolkit.repackage.readers.hf.make_hf_reader', fake_make_hf_reader), \ patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ - patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor): + patch.object(multiprocess, 'process_pool', InlineExecutor): session.return_value.client.return_value = FakeS3Client() if processes == 1: result = Repackager(source=source, prefix_path=str(tmp_path / 'out'), writer_info={}).run() @@ -476,7 +476,7 @@ def test_no_s3_client_without_s3(tmp_path): @pytest.mark.parametrize('remote', [False, True]) def test_multiprocess_repackage(tmp_path, remote): prefix = 'memory://mp-test/out/RUN' if remote else str(tmp_path / 'RUN') - with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ + with patch.object(multiprocess, 'process_pool', InlineExecutor), \ patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): n = multiprocess.run_multiprocess_repackage( source=RecordsSource(CAPTURES, 'https://data.example.org'), n_processes=3, readers_per_process=2, @@ -507,7 +507,7 @@ def test_multiprocess_repackage(tmp_path, remote): def test_multiprocess_repackage_keeps_the_order_of_the_source(tmp_path): """Workers do not sort their shard again (--no-sort streams, a sorted source is sorted already).""" captures = CAPTURES[::-1] - with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ + with patch.object(multiprocess, 'process_pool', InlineExecutor), \ patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): multiprocess.run_multiprocess_repackage( source=RecordsSource(captures, 'https://data.example.org', sort=False), n_processes=1, @@ -528,7 +528,7 @@ def fake_fetch_wb_warc(capture, wb): captures = [{'url': f'https://example.com/{i}', 'timestamp': '20240101000000', 'status': '200'} for i in range(3)] fs = fsspec.filesystem('memory') - with patch.object(multiprocess, 'ProcessPoolExecutor', InlineExecutor), \ + with patch.object(multiprocess, 'process_pool', InlineExecutor), \ patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc', side_effect=fake_fetch_wb_warc): result = multiprocess.run_multiprocess_repackage( source=WaybackSource(captures, 'ia'), n_processes=1, readers_per_process=1, diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py index 13fce18..9e19e75 100644 --- a/tests/unit/test_aws.py +++ b/tests/unit/test_aws.py @@ -238,3 +238,31 @@ def test_fsspec_retries_of_the_user_win(no_user_config, monkeypatch, retries, ex monkeypatch.setattr(fsspec.config, 'conf', {'s3': {'config_kwargs': {'retries': retries}}}) aws.configure_fsspec_s3() assert fsspec.config.conf['s3']['config_kwargs']['retries'] == expected + + +def fsspec_s3_conf(): + import fsspec.config + + return fsspec.config.conf.get('s3', {}) + + +@pytest.mark.parametrize('defaults', ['on', 'off']) +def test_spawned_processes_use_the_defaults(no_user_config, monkeypatch, defaults): + """A spawned process does not inherit the configuration of this one: process_pool() configures it.""" + import multiprocessing + from concurrent.futures import ProcessPoolExecutor + + monkeypatch.setenv('CDXT_AWS_DEFAULTS', defaults) + monkeypatch.setenv('FSSPEC_CONFIG_DIR', '/nonexistent') # not the fsspec configuration of the machine + ctx = multiprocessing.get_context('spawn') + with aws.process_pool(1, mp_context=ctx) as ex: + conf = ex.submit(fsspec_s3_conf).result() + if defaults == 'on': + assert conf['config_kwargs']['retries'] == FULL_DEFAULTS['retries'] + assert conf['config_kwargs']['s3'] == FULL_DEFAULTS['s3'] + assert conf['max_concurrency'] == 2 + else: + assert 'config_kwargs' not in conf + + with ProcessPoolExecutor(1, mp_context=ctx) as ex: # without process_pool() + assert 'config_kwargs' not in ex.submit(fsspec_s3_conf).result() From 8283da1bc24fd5d113cc20ea851d58c48420eb8d Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 18:01:03 +0200 Subject: [PATCH 75/91] fix(myrequests): every attempt, also a retry, takes its turn in the per-host rate limiter The slot was reserved once per request, so with parallel readers a retry could start together with another reader's request. The limiter also uses time.monotonic() now. --- cdx_toolkit/myrequests.py | 9 ++++---- docs/settings.md | 4 ++-- tests/unit/test_myrequests.py | 42 +++++++++++++++++++++++++++++++++-- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/cdx_toolkit/myrequests.py b/cdx_toolkit/myrequests.py index 37725e2..f722d14 100644 --- a/cdx_toolkit/myrequests.py +++ b/cdx_toolkit/myrequests.py @@ -32,7 +32,8 @@ def dns_fatal(hostname) -> bool: class HostRateLimiter: """The minimum interval between the starts of requests to the same host, also when requests are made from - several threads (e.g. the parallel WARC readers of cdxt repackage).""" + several threads (e.g. the parallel WARC readers of cdxt repackage). Every attempt, also a retry, takes its + turn. Times are time.monotonic().""" def __init__(self, default_interval: float, intervals: Dict[str, float]): self.default_interval = default_interval @@ -55,7 +56,7 @@ def finish(self, hostname: str) -> None: """After a request (which may have taken long, with retries): the next one may start the minimum interval from now, or later if that time is reserved already.""" with self._lock: - next_fetch = time.time() + self.minimum_interval(hostname) + next_fetch = time.monotonic() + self.minimum_interval(hostname) self._next_fetch[hostname] = max(self._next_fetch.get(hostname, 0), next_fetch) @@ -100,7 +101,6 @@ def myrequests_get( raise_warning_after_n_errors = WARNING_AFTER_N_ERRORS hostname = urlparse(url).hostname - _wait_for_turn(hostname, stop) minimum_interval = RATE_LIMITER.minimum_interval(hostname) if params: @@ -124,6 +124,7 @@ def myrequests_get( while retry: if stop is not None and stop.is_set(): raise RequestCancelled() + _wait_for_turn(hostname, stop) # after the backoff of a retry: both the backoff and the interval hold try: LOGGER.debug('getting %s %r', url, params) resp = requests.get(url, params=params, headers=headers, timeout=(30.0, 30.0), allow_redirects=False) @@ -192,7 +193,7 @@ def myrequests_get( def _wait_for_turn(hostname: str, stop: Optional[threading.Event]) -> None: """Wait until a request to `hostname` may start (see HostRateLimiter).""" - now = time.time() + now = time.monotonic() dt = RATE_LIMITER.reserve(hostname, now) - now if dt > 0: if dt > 3.1: diff --git a/docs/settings.md b/docs/settings.md index 62ee58c..0bacc90 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -16,8 +16,8 @@ Reads from `s3://` (see above), `hf://`, and the queries of DuckDB and Athena do ### Minimum interval between requests Requests to the same host start at least a minimum interval apart, also when they are made from several -threads (e.g. `cdxt repackage --parallel-readers`). After a request, including its retries, the next one -waits the interval again. +threads (e.g. `cdxt repackage --parallel-readers`). Every attempt counts, also a retry: it waits for its +backoff, then for its turn. After a request, including its retries, the next one waits the interval again. | Host | Interval | Environment variable | |---|---|---| diff --git a/tests/unit/test_myrequests.py b/tests/unit/test_myrequests.py index 8ca7d8f..e5d072d 100644 --- a/tests/unit/test_myrequests.py +++ b/tests/unit/test_myrequests.py @@ -13,7 +13,10 @@ def test_get_without_error(): assert resp.status_code == 200 -def test_get_with_connection_error(caplog): +def test_get_with_connection_error(caplog, monkeypatch): + from cdx_toolkit import myrequests + + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'example.com', 0) # every attempt takes its turn with patch('requests.get') as mock_get: mock_get.side_effect = requests.exceptions.ConnectionError('Connection failed') @@ -51,7 +54,7 @@ def test_finish_spaces_the_next_request_from_the_end_of_the_previous_one(): from cdx_toolkit.myrequests import HostRateLimiter limiter = HostRateLimiter(10.0, {}) - with patch('cdx_toolkit.myrequests.time.time', return_value=2000.0): + with patch('cdx_toolkit.myrequests.time.monotonic', return_value=2000.0): limiter.finish('example.org') assert limiter.reserve('example.org', 2001.0) == 2010.0 @@ -68,3 +71,38 @@ def test_dns_error_is_only_fatal_for_unknown_hosts(monkeypatch, host, fatal): assert 'invalid hostname' in str(e.value) and get.call_count == 1 else: assert '2 failures' in str(e.value) and get.call_count == 2 # retried + + +def test_retries_take_their_turn(monkeypatch): + """Three parallel requests to a host, the first answered with a 503: its retry keeps the minimum interval + to the other requests too (it used to start together with the third one).""" + import threading + import time + from unittest.mock import Mock + + from cdx_toolkit import myrequests + + host = 'retry-turn.example.org' + interval = 0.1 + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, host, interval) + starts = [] + lock = threading.Lock() + + def get(url, **kwargs): + with lock: + starts.append(time.monotonic()) + first = len(starts) == 1 + return Mock(status_code=503 if first else 200, text='') + + with patch('requests.get', side_effect=get): + threads = [threading.Thread(target=myrequests_get, args=(f'https://{host}/{i}',)) for i in range(3)] + for thread in threads: + thread.start() + time.sleep(0.01) # the first thread takes the first turn + for thread in threads: + thread.join() + + assert len(starts) == 4 + starts.sort() + gaps = [b - a for a, b in zip(starts, starts[1:])] + assert min(gaps) > interval * 0.8, gaps From cc2902b5d061b3b40377711fde01b557f2c01613 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 18:06:47 +0200 Subject: [PATCH 76/91] fix(hf): require huggingface_hub>=1.7.2 for Storage Buckets, with a clear error for older versions 0.30 (the old minimum) and 1.5.0 (the Buckets API) treat hf://buckets/ paths as model repos; 1.6.0 and 1.7.0/1.7.1 raise EntryNotFoundError for a missing path, which broke the existing-output checks. Verified with the live HF tests on Python 3.9: 1.7.2 passes, 1.7.1 fails. An installed older version (which satisfied >=0.30) is now reported before any work (exit 2); installing the extra upgrades it. New live test: cdxt index -o hf://. --- CHANGELOG.md | 3 +- cdx_toolkit/cli/__init__.py | 6 ++++ cdx_toolkit/hf.py | 33 +++++++++++++++++++++ cdx_toolkit/indexes/backends/duckdb.py | 3 ++ cdx_toolkit/repackage/readers/hf.py | 4 +++ requirements.txt | 2 +- setup.py | 2 +- tests/repackage/test_hf_live.py | 25 ++++++++++++++++ tests/unit/test_duckdb.py | 2 +- tests/unit/test_hf.py | 40 ++++++++++++++++++++++++++ 10 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 cdx_toolkit/hf.py create mode 100644 tests/unit/test_hf.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 45f3722..7bb0783 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,7 +44,8 @@ + Python API of the new commands: `cdx_toolkit.indexes.open_index()` with `IndexQuery` (`records()`, `count()`, `explain()`) and `cdx_toolkit.repackage.repackage()`; `cdxt index` and `cdxt repackage` run the same code + deprecated `cdx_toolkit.warc.get_writer()` and `CDXToolkitWARCWriter` (FutureWarning): use `repackage()` - + new extras: `athena`, `duckdb`, `hf`, `parquet`; `fsspec` and `aiofiles` are now required + + new extras: `athena`, `duckdb`, `hf` (`huggingface_hub>=1.7.2`, the first version whose `HfFileSystem` handles + Storage Buckets), `parquet`; `fsspec` and `aiofiles` are now required - 0.9.38 + deprecated support for py3.7 and py.3.8 diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py index dbc0e66..4f58d77 100644 --- a/cdx_toolkit/cli/__init__.py +++ b/cdx_toolkit/cli/__init__.py @@ -2,6 +2,7 @@ import sys from cdx_toolkit.aws import configure_fsspec_s3 +from cdx_toolkit.hf import check_huggingface_hub, uses_hf from cdx_toolkit.indexes import UsageError from cdx_toolkit.utils import get_version @@ -38,6 +39,11 @@ def main(args=None): set_loglevel(cmd.verbose) configure_fsspec_s3() # Common Crawl's S3 defaults for s3:// paths, unless configured by the user cmdline = ' '.join(['cdxt'] + list(args)) + if uses_hf(args): + try: + check_huggingface_hub() # before any work, not when the first hf:// path is opened + except RuntimeError as e: + parser.exit(2, f'cdxt {cmd.cmd}: error: {e}\n') try: cmd.func(cmd, cmdline) except (UsageError, FileNotFoundError) as e: diff --git a/cdx_toolkit/hf.py b/cdx_toolkit/hf.py new file mode 100644 index 0000000..049df76 --- /dev/null +++ b/cdx_toolkit/hf.py @@ -0,0 +1,33 @@ +''' +Hugging Face Storage Buckets (hf://buckets/...): the version of huggingface_hub they need. +''' +import re +from typing import Tuple + +# HfFileSystem lists and opens bucket paths since 1.6.0; before 1.7.2, listing a missing path raised +# EntryNotFoundError instead of FileNotFoundError (which the checks of existing outputs rely on) +HF_HUB_MIN_VERSION = (1, 7, 2) + + +def version_tuple(version: str) -> Tuple[int, ...]: + """'1.7.2' -> (1, 7, 2), '2.0.0rc1' -> (2, 0, 0)""" + return tuple(int(re.match(r'\d*', part).group() or 0) for part in version.split('.')[:3]) + + +def check_huggingface_hub() -> None: + """Raise RuntimeError with a clear message if huggingface_hub is missing or too old for hf://buckets/ paths + (an installed older version also satisfies a requirement without a minimum).""" + minimum = '.'.join(map(str, HF_HUB_MIN_VERSION)) + install = f"pip install 'cdx_toolkit[hf]' (or 'huggingface_hub>={minimum}')" + try: + import huggingface_hub + except ImportError: + raise RuntimeError(f'hf://buckets/ paths need huggingface_hub: {install}') from None + version = huggingface_hub.__version__ + if version_tuple(version) < HF_HUB_MIN_VERSION: + raise RuntimeError(f'hf://buckets/ paths need huggingface_hub >= {minimum}, not {version}: {install}') + + +def uses_hf(args) -> bool: + """Whether command line arguments name a Hugging Face bucket.""" + return any('hf://' in arg or arg == 'cc-url-index-hf' for arg in args) diff --git a/cdx_toolkit/indexes/backends/duckdb.py b/cdx_toolkit/indexes/backends/duckdb.py index 4d7462c..d1ced67 100644 --- a/cdx_toolkit/indexes/backends/duckdb.py +++ b/cdx_toolkit/indexes/backends/duckdb.py @@ -77,6 +77,9 @@ def iter_duckdb_rows(query: str, region_name: Optional[str] = None) -> Iterator[ _configure_httpfs(con, s3='s3://' in query, region_name=region_name) if 'hf://buckets/' in query: # DuckDB cannot read Hugging Face Storage Buckets itself; use their fsspec filesystem. + from cdx_toolkit.hf import check_huggingface_hub + + check_huggingface_hub() from huggingface_hub import HfFileSystem con.register_filesystem(HfFileSystem()) diff --git a/cdx_toolkit/repackage/readers/hf.py b/cdx_toolkit/repackage/readers/hf.py index 7a97c32..2aaeb86 100644 --- a/cdx_toolkit/repackage/readers/hf.py +++ b/cdx_toolkit/repackage/readers/hf.py @@ -258,6 +258,10 @@ async def make_hf_reader(max_attempts: int = 5, base_backoff_seconds: float = 0. reading from a bucket. """ import aiohttp + + from cdx_toolkit.hf import check_huggingface_hub + + check_huggingface_hub() from huggingface_hub import get_token timeout = aiohttp.ClientTimeout(total=300, sock_connect=15) diff --git a/requirements.txt b/requirements.txt index a1150de..f40d819 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ duckdb pytz # optional Hugging Face dependencies (install via cdx_toolkit[hf]) -huggingface_hub>=0.30 +huggingface_hub>=1.7.2 aiohttp # optional Parquet dependencies (install via cdx_toolkit[parquet]) diff --git a/setup.py b/setup.py index 39e9b9e..22fc5ec 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ # `cdxt index cc-url-index-s3` and other parquet indexes optional_duckdb_requirements = ['duckdb', 'pytz'] # pytz: DuckDB returns timestamps with time zone # Hugging Face Storage Buckets (hf://): `cdxt index cc-url-index-hf`, `cdxt repackage` reads and writes -optional_hf_requirements = ['huggingface_hub>=0.30', 'aiohttp'] +optional_hf_requirements = ['huggingface_hub>=1.7.2', 'aiohttp'] # Parquet output of `cdxt index` (and Parquet input of `cdxt repackage`) optional_parquet_requirements = ['pyarrow'] diff --git a/tests/repackage/test_hf_live.py b/tests/repackage/test_hf_live.py index ef1856d..dc20cab 100644 --- a/tests/repackage/test_hf_live.py +++ b/tests/repackage/test_hf_live.py @@ -8,6 +8,7 @@ import os import fsspec +import pytest from warcio.archiveiterator import ArchiveIterator from cdx_toolkit.cli import main @@ -99,3 +100,27 @@ def test_repackage_from_hf_parallel_readers(tmpdir): HF_PREFIX, '--parallel-readers', '3']) assert _responses(os.path.join(str(tmpdir), 'TEST_hf-001.warc.gz')) == _expected() + + +@requires_live_hf +def test_index_output_to_hf_bucket(tmpdir, hf_tmpdir, capsys): + """cdxt index writes to a bucket: a single output, and one output per CDX file, which needs a new or empty + directory (also in a bucket).""" + cdx = os.path.join(str(tmpdir), 'in', 'index.cdx') + os.makedirs(os.path.dirname(cdx)) + with open(cdx, 'w') as f: + for capture in CAPTURES: + f.write(f'org,commoncrawl)/ {capture["timestamp"]} {json.dumps(capture)}\n') + + main(args=['index', cdx, '--domain', 'commoncrawl.org', '-o', f'{hf_tmpdir}/captures.jsonl']) + with fsspec.open(f'{hf_tmpdir}/captures.jsonl', 'rt') as f: + assert [json.loads(line)['url'] for line in f] == [c['url'] for c in CAPTURES] + + main(args=['index', os.path.join(str(tmpdir), 'in', '*.cdx'), '--domain', 'commoncrawl.org', + '-o', f'{hf_tmpdir}/filtered']) + with fsspec.open(f'{hf_tmpdir}/filtered/index.cdx', 'rt') as f: + assert len(f.read().splitlines()) == len(CAPTURES) + with pytest.raises(SystemExit): + main(args=['index', os.path.join(str(tmpdir), 'in', '*.cdx'), '--domain', 'commoncrawl.org', + '-o', f'{hf_tmpdir}/filtered']) + assert 'is not empty' in capsys.readouterr().err diff --git a/tests/unit/test_duckdb.py b/tests/unit/test_duckdb.py index 97e898e..569f9b9 100644 --- a/tests/unit/test_duckdb.py +++ b/tests/unit/test_duckdb.py @@ -77,7 +77,7 @@ def test_connection_setup_and_hf_filesystem(monkeypatch): fake_duckdb = MagicMock(connect=MagicMock(return_value=con)) hf_fs = MagicMock() - fake_hf = MagicMock(HfFileSystem=MagicMock(return_value=hf_fs)) + fake_hf = MagicMock(HfFileSystem=MagicMock(return_value=hf_fs), __version__='1.7.2') query = "SELECT count(*) AS n FROM read_parquet(['hf://buckets/x/y/*.parquet', 's3://commoncrawl/x.parquet'])" with patch.object(duckdb_source, '_HAS_DUCKDB', True), \ patch.object(duckdb_source, 'duckdb', fake_duckdb, create=True), \ diff --git a/tests/unit/test_hf.py b/tests/unit/test_hf.py new file mode 100644 index 0000000..e947c3b --- /dev/null +++ b/tests/unit/test_hf.py @@ -0,0 +1,40 @@ +import sys +from unittest.mock import patch + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.hf import check_huggingface_hub, uses_hf, version_tuple + + +def test_version_tuple(): + assert version_tuple('1.7.2') == (1, 7, 2) + assert version_tuple('2.0.0rc1') == (2, 0, 0) + assert version_tuple('1.10') == (1, 10) + assert version_tuple('1.7.2') >= (1, 7, 2) > version_tuple('1.7.1') > version_tuple('0.30.0') + + +@pytest.mark.parametrize('version, ok', [('0.30.0', False), ('1.7.1', False), ('1.7.2', True), ('2.0.0', True)]) +def test_check_huggingface_hub(version, ok): + with patch.dict(sys.modules, {'huggingface_hub': type(sys)('huggingface_hub')}): + sys.modules['huggingface_hub'].__version__ = version + if ok: + check_huggingface_hub() + else: + with pytest.raises(RuntimeError, match=r'huggingface_hub >= 1\.7\.2'): + check_huggingface_hub() + + +def test_missing_huggingface_hub(): + with patch.dict(sys.modules, {'huggingface_hub': None}), pytest.raises(RuntimeError, match=r'cdx_toolkit\[hf\]'): + check_huggingface_hub() + + +def test_cli_checks_the_version_first(capsys): + assert uses_hf(['index', 'cc-url-index-hf']) and uses_hf(['repackage', 'x', '--prefix', 'hf://buckets/a/b/c']) + assert not uses_hf(['index', 'cc', '--url', 'example.com/*']) + with patch.dict(sys.modules, {'huggingface_hub': type(sys)('huggingface_hub')}), pytest.raises(SystemExit) as e: + sys.modules['huggingface_hub'].__version__ = '0.30.0' + main(['index', 'cc-url-index-hf', '--domain', 'example.com', '--dry-run']) + assert e.value.code == 2 + assert 'huggingface_hub >= 1.7.2, not 0.30.0' in capsys.readouterr().err From e61b1a28178917049ef9495b07bc42ffd46ea186 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 18:08:02 +0200 Subject: [PATCH 77/91] fix(indexes): --cc-mirror and --cc-sort are rejected for other indexes than cc They were passed to open_index() and silently ignored by the CDX files, DuckDB and Athena backends, and by ia and custom CDX API URLs (CDXFetcher only uses them for cc). --- README.md | 4 +++- cdx_toolkit/indexes/resolve.py | 7 ++++++- tests/unit/test_usage_errors.py | 20 +++++++++++++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 36e2a2d..e3800fe 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,9 @@ or `*.example.com`, or an exact URL), `--hostname` (an exact hostname like These can be repeated and combined; a capture is selected if it matches any of them. All other options, like `--crawl`, `--from`, `--to`, `--filter` and `--url-fgrep`, narrow the selection down. -Options an index does not support (e.g. `--filter` with DuckDB) are rejected. +Options an index does not support (e.g. `--filter` with DuckDB, or `--cc-mirror` +and `--cc-sort` with any other index than `cc`) are rejected; `--yes` is +accepted by every index (only the SQL indexes ask for confirmation). For CDX files and the SQL indexes, `--url` patterns are matched against SURT keys (`url_surtkey` in the columnar index), like the CDX API does: `www.` is ignored, so `--url 'example.com/*'` also matches `www.example.com`, while diff --git a/cdx_toolkit/indexes/resolve.py b/cdx_toolkit/indexes/resolve.py index 3fd35db..1da1e1e 100644 --- a/cdx_toolkit/indexes/resolve.py +++ b/cdx_toolkit/indexes/resolve.py @@ -58,12 +58,17 @@ def open_index( - anything else (local or remote CDX files, globs) -> CDX files `backend` ('cdx-api', 'cdx-files', 'duckdb' or 'athena') overrides the detection, e.g. 'cdx-files' for a CDX - file at an http(s) URL without file extension. + file at an http(s) URL without file extension. `cc_mirror` and `cc_sort` are options of the Common Crawl CDX + API ('cc') and rejected for other indexes; `assume_yes` is accepted by all (only SQL indexes ask). """ if backend not in INDEX_BACKENDS: raise UsageError(f'unknown index backend {backend!r} (choose from {", ".join(INDEX_BACKENDS)})') if backend == 'auto': backend = detect_index_backend(location) + if (cc_mirror is not None or cc_sort is not None) and not (backend == 'cdx-api' and location == 'cc'): + options = ' and '.join(name for name, value in (('--cc-mirror', cc_mirror), ('--cc-sort', cc_sort)) + if value is not None) + raise UsageError(f'{options} can only be used with the Common Crawl CDX API (cc), not with {location}') location = INDEX_ALIASES.get(location, location) if backend == 'athena': diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py index cd73378..d29d575 100644 --- a/tests/unit/test_usage_errors.py +++ b/tests/unit/test_usage_errors.py @@ -7,7 +7,7 @@ import pytest from cdx_toolkit.cli import main -from cdx_toolkit.indexes import IndexQuery, UsageError +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend, load_url_patterns from cdx_toolkit.indexes.patterns import domain_entry_to_surts, url_pattern_to_surts @@ -120,3 +120,21 @@ def test_parquet_batches_exactly_full(tmp_path, monkeypatch): path = str(tmp_path / 'r.parquet') assert write_records(iter([{'url': str(i)} for i in range(4)]), path, 'parquet') == 4 assert [r['url'] for r in read_records(path)] == ['0', '1', '2', '3'] + + +@pytest.mark.parametrize('index', ['ia', 'https://example.org/cdx', 'index.cdx', 'cc-url-index-s3', 'x.parquet', + 'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://b/']) +@pytest.mark.parametrize('option', [['--cc-mirror', 'https://mirror.example.org/'], ['--cc-sort', 'ascending']]) +def test_cc_options_only_for_cc(capsys, index, option): + with pytest.raises(SystemExit) as e: + main(['index', index, '--domain', 'example.com', '--dry-run'] + option) + assert e.value.code == 2 + assert f'{option[0]} can only be used with the Common Crawl CDX API (cc)' in capsys.readouterr().err + + with pytest.raises(UsageError, match=option[0]): + open_index(index, **{option[0][2:].replace('-', '_'): option[1]}) + + +def test_cc_options_are_passed_on(): + backend = open_index('cc', cc_mirror='https://mirror.example.org/', cc_sort='ascending', assume_yes=True) + assert (backend.cc_mirror, backend.cc_sort) == ('https://mirror.example.org/', 'ascending') From aecab0a1825ca54dd12ddca205c473ae89f850a2 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 18:09:59 +0200 Subject: [PATCH 78/91] fix(indexes): --from, --to and --closest are validated as dates by every backend --to 202413 raised an uncaught IllegalMonthError for SQL indexes (the padding was outside the error handling), and CDX files accepted impossible dates such as 20240230. timeutils.timestamp_range() checks the padded first and last second of a timestamp; IndexBackend.validate() uses it for all backends (exit 2 before any request), and the SQL bounds are built from it. --- cdx_toolkit/indexes/backends/cdx_files.py | 3 -- cdx_toolkit/indexes/backends/sql_builder.py | 16 ++++----- cdx_toolkit/indexes/base.py | 8 +++++ cdx_toolkit/timeutils.py | 18 ++++++++++ tests/unit/test_cdx_files.py | 39 +++++++++++++++++++++ 5 files changed, 72 insertions(+), 12 deletions(-) diff --git a/cdx_toolkit/indexes/backends/cdx_files.py b/cdx_toolkit/indexes/backends/cdx_files.py index cba0ebd..432f1dc 100644 --- a/cdx_toolkit/indexes/backends/cdx_files.py +++ b/cdx_toolkit/indexes/backends/cdx_files.py @@ -206,9 +206,6 @@ def validate(self, query: IndexQuery) -> None: raise UsageError( f'select captures from the {self.name} with --url, --hostname, --domain, --url-file, --from or --to' ) - for timestamp in (query.from_ts, query.to): - if timestamp and not (timestamp.isdigit() and 4 <= len(timestamp) <= 14): - raise UsageError(f'invalid timestamp {timestamp!r}, use e.g. 2008, 20080328 or 200906302359') for pattern in query.urls: url_pattern_to_surts(pattern) diff --git a/cdx_toolkit/indexes/backends/sql_builder.py b/cdx_toolkit/indexes/backends/sql_builder.py index 6506daf..556a1d3 100644 --- a/cdx_toolkit/indexes/backends/sql_builder.py +++ b/cdx_toolkit/indexes/backends/sql_builder.py @@ -7,7 +7,7 @@ from typing import List, Optional, Tuple from cdx_toolkit.commoncrawl import normalize_crawl, get_cc_endpoints, match_cc_crawls -from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up +from cdx_toolkit.timeutils import timestamp_range from ..base import UsageError from ..patterns import domain_to_surts, normalize_host, url_pattern_to_surts @@ -213,15 +213,13 @@ def resolve_crawl_names(crawl_arg) -> List[str]: def sql_timestamp(timestamp: str, end: bool = False) -> str: """CDX timestamp (e.g. 2008, 200906302359) -> SQL TIMESTAMP literal of its first second, or with `end`, of the second after its last second (an exclusive end, which includes the fractions of the last second).""" - if not re.fullmatch(r'\d{4,14}', timestamp): - raise UsageError(f'invalid timestamp {timestamp!r}, use e.g. 2008, 20080328 or 200906302359') - padded = pad_timestamp_up(timestamp) if end else pad_timestamp(timestamp) try: - moment = datetime.datetime.strptime(padded, '%Y%m%d%H%M%S') - if end: - moment += datetime.timedelta(seconds=1) - except (ValueError, OverflowError): - raise UsageError(f'invalid timestamp {timestamp!r}') from None + first, last = timestamp_range(timestamp) + moment = last + datetime.timedelta(seconds=1) if end else first + except ValueError as e: + raise UsageError(str(e)) from None + except OverflowError: # the end of 9999 + raise UsageError(f'invalid timestamp {timestamp!r} (too late)') from None return f"TIMESTAMP '{moment:%Y-%m-%d %H:%M:%S}'" diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py index d009fa6..7259f0d 100644 --- a/cdx_toolkit/indexes/base.py +++ b/cdx_toolkit/indexes/base.py @@ -3,6 +3,8 @@ import logging from typing import FrozenSet, Iterable, Iterator, List, Mapping +from cdx_toolkit.timeutils import timestamp_range + from .query import IndexQuery, field_flag @@ -40,6 +42,12 @@ def validate(self, query: IndexQuery) -> None: raise UsageError(f'{options} cannot be used with the {self.name} index') if query.limit is not None and query.limit < 1: raise UsageError(f'--limit must be at least 1, not {query.limit}') + for option, timestamp in (('--from', query.from_ts), ('--to', query.to), ('--closest', query.closest)): + if timestamp is not None: + try: + timestamp_range(timestamp) + except ValueError as e: + raise UsageError(f'{option}: {e}') from None def input_locations(self, query: IndexQuery) -> List[str]: """The files (paths, directories or globs) the query reads, which an output must not overwrite.""" diff --git a/cdx_toolkit/timeutils.py b/cdx_toolkit/timeutils.py index 3540710..39c5bfd 100644 --- a/cdx_toolkit/timeutils.py +++ b/cdx_toolkit/timeutils.py @@ -38,6 +38,24 @@ def pad_timestamp_up(ts): return padded +def timestamp_range(ts): + """The first and the last second of a CDX timestamp prefix (4 to 14 digits of yyyyMMddhhmmss, padded like the + CDX API pads it) as datetimes; ValueError for an invalid one (e.g. month 13 or February 30th). + + >>> timestamp_range('202402') + (datetime.datetime(2024, 2, 1, 0, 0), datetime.datetime(2024, 2, 29, 23, 59, 59)) + """ + if not (isinstance(ts, str) and ts.isascii() and ts.isdigit() and 4 <= len(ts) <= 14): + raise ValueError(f'invalid timestamp {ts!r}, use e.g. 2008, 200803, 20080328 or 200906302359') + try: + return tuple( + datetime.datetime(*(int(padded[i:j]) for i, j in ((0, 4), (4, 6), (6, 8), (8, 10), (10, 12), (12, 14)))) + for padded in (pad_timestamp(ts), pad_timestamp_up(ts)) + ) + except ValueError: # also calendar.IllegalMonthError + raise ValueError(f'invalid timestamp {ts!r} (no such date or time)') from None + + def timestamp_to_time(ts): ''' >>> timestamp_to_time('1999') diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py index 631ceb9..3b927b4 100644 --- a/tests/unit/test_cdx_files.py +++ b/tests/unit/test_cdx_files.py @@ -317,3 +317,42 @@ def test_output_checks_on_s3(s3_tmpdir, capsys): main(['index', f'{s3_tmpdir}/in/*.cdx.gz', '--domain', 'mnhn.fr', '-o', f'{s3_tmpdir}/out2']) assert fs.exists(f'{s3_tmpdir}/out2/a.cdx.gz') + + +@pytest.mark.parametrize('timestamp, valid', [ + ('2024', True), ('202402', True), ('20240229', True), ('2024022923', True), ('202402292359', True), + ('20240229235959', True), ('2024080123595', True), + ('202400', False), ('202413', False), ('20230229', False), ('20240230', False), ('20240231', False), + ('2024022924', False), ('202402292360', False), ('20245', False), ('2024-01', False), ('202', False), + ('202402292359590', False), ('２０２４', False), +]) +def test_timestamps_are_validated_by_every_backend(capsys, timestamp, valid): + """Invalid calendar dates are rejected (exit 2, no traceback) before a query, by CDX files, SQL and the API.""" + for index in (str(CDX_PATH), 'cc-url-index-s3', 'cc'): + for option in ('--from', '--to'): + args = ['index', index, '--domain', 'example.com', option, timestamp, '--dry-run'] + if valid: + main(args) + capsys.readouterr() + else: + with pytest.raises(SystemExit) as e: + main(args) + assert e.value.code == 2 + err = capsys.readouterr().err + assert f'{option}: invalid timestamp' in err and 'Traceback' not in err, (index, option) + + +def test_cdx_files_and_sql_select_the_same_time_range(tmp_path, capsys): + """Leap day and end of month: the bounds of the file backend (string comparison) and SQL are the same.""" + pytest.importorskip('duckdb') + from cdx_toolkit.indexes.backends.sql_builder import sql_timestamp + + assert sql_timestamp('202402') == "TIMESTAMP '2024-02-01 00:00:00'" + assert sql_timestamp('202402', end=True) == "TIMESTAMP '2024-03-01 00:00:00'" + lines = [f'com,example)/ {ts} {json.dumps({"url": "https://example.com/", "offset": ts})}\n' + for ts in ('20240131235959', '20240201000000', '20240229235959', '20240301000000')] + path = tmp_path / 'index.cdx' + path.write_text(''.join(lines)) + main(['index', str(path), '--domain', 'example.com', '--from', '202402', '--to', '202402', '--fields', 'offset', + '--output-format', 'csv']) + assert capsys.readouterr().out.split() == ['offset', '20240201000000', '20240229235959'] From ac9ce043d20afd872a00494d5355d2946f569702 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 18:14:17 +0200 Subject: [PATCH 79/91] docs: quote the extras in install commands (zsh treats cdx_toolkit[s3] as a glob) Also in the error messages that suggest an install, and in the CI workflows. --- .github/workflows/ci-slow.yaml | 2 +- .github/workflows/ci.yaml | 4 ++-- README.md | 12 ++++++------ cdx_toolkit/indexes/backends/duckdb.py | 2 +- cdx_toolkit/record_io.py | 2 +- cdx_toolkit/warc.py | 3 ++- docs/usage/aws-athena-s3.md | 2 +- docs/usage/http.md | 2 +- docs/usage/huggingface.md | 2 +- 9 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml index 86d64bd..5c3f433 100644 --- a/.github/workflows/ci-slow.yaml +++ b/.github/workflows/ci-slow.yaml @@ -50,7 +50,7 @@ jobs: pip install setuptools - name: Install cdx_toolkit - run: pip install .[all] + run: pip install ".[all]" - name: Configure AWS credentials from OIDC uses: aws-actions/configure-aws-credentials@v6 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 27479a4..a9a524f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,7 +67,7 @@ jobs: pip install setuptools - name: Install cdx_toolkit - run: pip install .[all] + run: pip install ".[all]" - name: Set environment variables for faster unit tests (requests are mocked) uses: actions/github-script@v9 @@ -125,7 +125,7 @@ jobs: pip install setuptools - name: Install cdx_toolkit (minimal) - run: pip install .[test] + run: pip install ".[test]" - name: Set environment variables for faster unit tests (requests are mocked) uses: actions/github-script@v9 diff --git a/README.md b/README.md index e3800fe..1172d6f 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,12 @@ or clone this repo and use `pip install .` Optional extras: ``` -$ pip install cdx_toolkit[s3] # S3 input/output and WARC downloads from s3:// -$ pip install cdx_toolkit[duckdb] # query the columnar URL index with DuckDB -$ pip install cdx_toolkit[athena] # query the columnar URL index with AWS Athena -$ pip install cdx_toolkit[hf] # Hugging Face Storage Buckets (hf://) -$ pip install cdx_toolkit[parquet] # Parquet output (and input of cdxt repackage) -$ pip install cdx_toolkit[all] # install all optional dependencies +$ pip install 'cdx_toolkit[s3]' # S3 input/output and WARC downloads from s3:// +$ pip install 'cdx_toolkit[duckdb]' # query the columnar URL index with DuckDB +$ pip install 'cdx_toolkit[athena]' # query the columnar URL index with AWS Athena +$ pip install 'cdx_toolkit[hf]' # Hugging Face Storage Buckets (hf://) +$ pip install 'cdx_toolkit[parquet]' # Parquet output (and input of cdxt repackage) +$ pip install 'cdx_toolkit[all]' # install all optional dependencies ``` ### Settings diff --git a/cdx_toolkit/indexes/backends/duckdb.py b/cdx_toolkit/indexes/backends/duckdb.py index d1ced67..19a5b87 100644 --- a/cdx_toolkit/indexes/backends/duckdb.py +++ b/cdx_toolkit/indexes/backends/duckdb.py @@ -63,7 +63,7 @@ def iter_duckdb_rows(query: str, region_name: Optional[str] = None) -> Iterator[ public bucket is readable with valid credentials.""" if not _HAS_DUCKDB: raise RuntimeError( - 'DuckDB engine requires optional dependencies. Install cdx_toolkit[duckdb].' + "DuckDB engine requires optional dependencies. Install them with: pip install 'cdx_toolkit[duckdb]'" ) logger.info('Executing DuckDB query: %s', query) diff --git a/cdx_toolkit/record_io.py b/cdx_toolkit/record_io.py index 72f8296..2a26e69 100644 --- a/cdx_toolkit/record_io.py +++ b/cdx_toolkit/record_io.py @@ -251,7 +251,7 @@ def import_pyarrow(): import pyarrow.parquet except ImportError: raise RuntimeError( - 'Parquet files need the optional pyarrow dependency. Install cdx_toolkit[parquet].' + "Parquet files need the optional pyarrow dependency. Install it with: pip install 'cdx_toolkit[parquet]'" ) from None return pyarrow diff --git a/cdx_toolkit/warc.py b/cdx_toolkit/warc.py index c8a79db..06a9026 100644 --- a/cdx_toolkit/warc.py +++ b/cdx_toolkit/warc.py @@ -29,7 +29,8 @@ def _is_s3_url(url): def _require_s3_deps(): if not _HAS_FSSPEC: raise RuntimeError( - 'Remote filesystem (S3) support requires optional dependencies. Install cdx_toolkit[s3].' + "Remote filesystem (S3) support requires optional dependencies. " + "Install them with: pip install 'cdx_toolkit[s3]'" ) diff --git a/docs/usage/aws-athena-s3.md b/docs/usage/aws-athena-s3.md index 98c8bb2..e6c9d62 100644 --- a/docs/usage/aws-athena-s3.md +++ b/docs/usage/aws-athena-s3.md @@ -9,7 +9,7 @@ selections, especially when `cdxt repackage` runs in `us-east-1` (e.g. on EC2), You need: -- `pip install cdx_toolkit[athena,s3]` (and `[duckdb]` for DuckDB); +- `pip install 'cdx_toolkit[athena,s3]'` (and `[duckdb]` for DuckDB); - AWS credentials, found like boto3 finds them (environment, `~/.aws` profiles, SSO, instance roles). Reads from `s3://commoncrawl` need credentials of any AWS account; anonymous requests are refused; - for Athena: an S3 bucket of your own for the query results (`S3OutputLocation`), and the table of the diff --git a/docs/usage/http.md b/docs/usage/http.md index a754242..b017784 100644 --- a/docs/usage/http.md +++ b/docs/usage/http.md @@ -16,7 +16,7 @@ columnar index (below) or, faster, [AWS](aws-athena-s3.md) or [Hugging Face](hug ```sh pip install cdx_toolkit # CDX API and HTTP downloads -pip install cdx_toolkit[duckdb] # also the columnar index with DuckDB +pip install 'cdx_toolkit[duckdb]' # also the columnar index with DuckDB ``` ## Small scale diff --git a/docs/usage/huggingface.md b/docs/usage/huggingface.md index f12398d..f410a0b 100644 --- a/docs/usage/huggingface.md +++ b/docs/usage/huggingface.md @@ -9,7 +9,7 @@ files to a bucket of your own, and run as a [Hugging Face Job](https://huggingfa next to the data. ```sh -pip install cdx_toolkit[hf,duckdb] +pip install 'cdx_toolkit[hf,duckdb]' hf auth login # or set HF_TOKEN ``` From 4716df1b42862c793468dd0ca875e2dab15b893d Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 18:16:35 +0200 Subject: [PATCH 80/91] docs: examples/homepages.sql, the --sql query of the examples The README, the help texts and the migration guide used a homepages.sql that was nowhere: the homepages of .va in one crawl, with the columns cdxt repackage needs; tested on a local copy of the index's columns, and run once on s3://commoncrawl (50 captures in 41 s). --- README.md | 6 +++-- cdx_toolkit/cli/index.py | 2 +- cdx_toolkit/cli/repackage.py | 2 +- docs/cli-migration-guide.md | 4 ++-- examples/homepages.sql | 15 ++++++++++++ tests/unit/test_sql_indexes.py | 43 +++++++++++++++++++++++++++++++++- 6 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 examples/homepages.sql diff --git a/README.md b/README.md index 1172d6f..6ac7ea1 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ $ cdxt index cc --url 'commoncrawl.org/*' --count # estimated number of capture $ cdxt index ia --url 'commoncrawl.org/*' --limit 10 # will show the beginning of IA's crawl $ cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.csv # DuckDB -$ cdxt index cc-url-index-s3 --sql homepages.sql --yes -o captures.csv +$ cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o captures.csv $ cdxt index cc --url 'commoncrawl.org/*' --limit 10 -o captures.jsonl $ cdxt repackage captures.jsonl --prefix COMMONCRAWL @@ -133,7 +133,9 @@ is one of the given [ISO 639-3](https://iso639-3.sil.org/) codes, e.g. `eng`, The columnar index can also be queried with your own SQL: `--sql query.sql` (or `--sql -` for stdin) runs the query as it is, instead of `--hostname`, -`--domain`, `--crawl`, `--from`, `--to`, `--language` and `--limit`. Without `--sql`, the query selects the +`--domain`, `--crawl`, `--from`, `--to`, `--language` and `--limit`; the query names the +Parquet files it reads, see [examples/homepages.sql](examples/homepages.sql), and the +[usage guides](docs/usage/) for more. Without `--sql`, the query selects the CDX fields (`url`, `timestamp`, `status`, `mime`, `digest`, `filename`, `offset`, `length`, `languages`, ... and `crawl`) for the given hosts and domains; `--dry-run` prints it without running it. diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py index b9cf197..8b76f93 100644 --- a/cdx_toolkit/cli/index.py +++ b/cdx_toolkit/cli/index.py @@ -36,7 +36,7 @@ cdxt index ia --url 'commoncrawl.org/*' --from 2008 --to 200906302359 -o captures.jsonl cdxt index cc --url 'commoncrawl.org/*' --count cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.jsonl - cdxt index cc-url-index-s3 --sql homepages.sql --yes -o captures.csv + cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o captures.csv # in the cdx_toolkit repo cdxt index 'indexes/cdx-*.gz' --url-file urls.txt -o filtered/ # one output per CDX file cdxt index cc --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --prefix COMMONCRAWL ''' diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py index a884f3a..01a4688 100644 --- a/cdx_toolkit/cli/repackage.py +++ b/cdx_toolkit/cli/repackage.py @@ -28,7 +28,7 @@ cdxt index cc --url 'commoncrawl.org/*' --limit 10 -o captures.jsonl cdxt repackage captures.jsonl --prefix COMMONCRAWL --size 100MB - cdxt index cc-url-index-s3 --sql homepages.sql --yes -o homepages.csv + cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o homepages.csv # in the cdx_toolkit repo cdxt repackage homepages.csv --prefix s3://my-bucket/HOMEPAGES --warc-download-prefix s3://commoncrawl \\ --processes 8 --parallel-readers 16 diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index ff9ac4a..a5229a2 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -260,8 +260,8 @@ or AWS Athena. This was not possible with the old commands: # DuckDB over s3://commoncrawl/cc-index/table/cc-main/warc/ (needs AWS credentials and cdx_toolkit[duckdb]) cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.jsonl -# your own SQL query -cdxt index cc-url-index-s3 --sql homepages.sql --yes -o homepages.csv +# your own SQL query (examples/homepages.sql in the cdx_toolkit repository) +cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o homepages.csv # AWS Athena (needs cdx_toolkit[athena]) cdxt index 'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://my-bucket/athena/' \ diff --git a/examples/homepages.sql b/examples/homepages.sql new file mode 100644 index 0000000..b8c95fb --- /dev/null +++ b/examples/homepages.sql @@ -0,0 +1,15 @@ +-- The homepages of the .va domain in one crawl: successful captures of the path "/" without a query string. +-- +-- cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o homepages.csv +-- cdxt repackage homepages.csv --prefix HOMEPAGES +-- +-- --sql runs the query as it is, so it names the Parquet files it reads (here one crawl of the columnar URL +-- index on S3, which needs AWS credentials); INDEX (cc-url-index-s3) only selects DuckDB. cdxt repackage needs +-- the columns url, warc_filename, warc_record_offset and warc_record_length. +SELECT url, fetch_time, fetch_status, content_mime_type, + warc_filename, warc_record_offset, warc_record_length +FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-33/subset=warc/*.parquet') +WHERE url_host_tld = 'va' + AND url_path = '/' + AND url_query IS NULL + AND fetch_status = 200 diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py index 135945c..ceb09cb 100644 --- a/tests/unit/test_sql_indexes.py +++ b/tests/unit/test_sql_indexes.py @@ -12,8 +12,9 @@ from cdx_toolkit.indexes.backends.duckdb import DuckDbBackend from cdx_toolkit.indexes.backends.sql import normalize_row from cdx_toolkit.indexes.backends.sql_builder import check_language, query_conditions +from cdx_toolkit.record_io import read_records -from tests.conftest import requires_duckdb +from tests.conftest import TEST_DATA_PATH, requires_duckdb ATHENA = 'athena://athena.us-east-1.amazonaws.com:443/ccindex?S3OutputLocation=s3://my-bucket/athena/' @@ -379,3 +380,43 @@ def test_end_timestamp_includes_fractional_seconds(to, included): def test_end_timestamp_overflow(): with pytest.raises(UsageError, match='invalid timestamp'): query_conditions(IndexQuery(to='9999')) + + +@requires_duckdb +def test_example_homepages_sql(tmp_path): + """examples/homepages.sql (the --sql example of the README), on a local copy of the index's columns.""" + import duckdb + + sql = (TEST_DATA_PATH.parent.parent / 'examples' / 'homepages.sql').read_text() + s3_files = 's3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-33/subset=warc/*.parquet' + assert s3_files in sql + rows = [ + # url, url_host_tld, url_path, url_query, fetch_status + ('https://www.vatican.va/', 'va', '/', None, 200), # a homepage + ('https://www.vatican.va/content/', 'va', '/content/', None, 200), + ('https://www.vatican.va/?lang=it', 'va', '/', 'lang=it', 200), + ('http://vatican.va/', 'va', '/', None, 301), + ('https://example.com/', 'com', '/', None, 200), + ] + values = ', '.join( + f"('{url}', '{tld}', '{path}', {repr(query) if query else 'NULL'}, {status}, 'text/html', " + f"TIMESTAMP '2024-08-01 12:00:00', 'crawl-data/a.warc.gz', {i}, 100)" + for i, (url, tld, path, query, status) in enumerate(rows) + ) + index = tmp_path / 'part-0.parquet' + duckdb.connect().execute(f''' + COPY (SELECT * FROM (VALUES {values}) AS t(url, url_host_tld, url_path, url_query, fetch_status, + content_mime_type, fetch_time, warc_filename, warc_record_offset, warc_record_length)) + TO '{index}' (FORMAT parquet) + ''') + query = tmp_path / 'homepages.sql' + query.write_text(sql.replace(s3_files, str(index))) + out = tmp_path / 'homepages.csv' + + main(['index', str(index), '--sql', str(query), '--yes', '-o', str(out)]) + + captures = list(read_records(str(out), 'csv')) + assert [c['url'] for c in captures] == ['https://www.vatican.va/'] + # the fields cdxt repackage needs + assert {k: captures[0][k] for k in ('filename', 'offset', 'length')} == { + 'filename': 'crawl-data/a.warc.gz', 'offset': '0', 'length': '100'} From e7ebbcc44dfb637b0dec9b2c38f1882d6c234608 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 18:16:35 +0200 Subject: [PATCH 81/91] refactor(repackage): one formatter of the WARC file names; current docstrings _output_filename split an S3 URL only to rebuild it (generate_warc_filename takes the whole prefix); removed the stale TODO about .extracted.warc and the docstring that said subprefixes distinguish the processes. --- cdx_toolkit/repackage/multiprocess.py | 17 +++------------ cdx_toolkit/repackage/writers/series.py | 10 +++++---- tests/repackage/test_multiprocess_naming.py | 24 ++++++++++----------- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py index 3c9c48d..5f29ff8 100644 --- a/cdx_toolkit/repackage/multiprocess.py +++ b/cdx_toolkit/repackage/multiprocess.py @@ -42,7 +42,7 @@ from cdx_toolkit.record_io import RecordWriter, read_records from cdx_toolkit.repackage.jobs import RecordsSource, RepackageResult, WaybackJob, WaybackSource from cdx_toolkit.repackage.repackager import Repackager -from cdx_toolkit.repackage.urls import is_s3_url, parse_s3_uri +from cdx_toolkit.repackage.urls import is_s3_url from cdx_toolkit.repackage.writers.series import generate_warc_filename logger = logging.getLogger(__name__) @@ -58,16 +58,6 @@ _CANCEL = None -def _output_filename(prefix_path: str, sequence: int, subprefix: Optional[str] = None, - gzip: bool = True) -> str: - """Build a WARC output path for a given rotation sequence (S3 uri or local path).""" - if is_s3_url(prefix_path): - bucket, key_prefix = parse_s3_uri(prefix_path) - key = generate_warc_filename(key_prefix, sequence=sequence, writer_subprefix=subprefix, gzip=gzip) - return f's3://{bucket}/{key}' - return generate_warc_filename(prefix_path, sequence=sequence, writer_subprefix=subprefix, gzip=gzip) - - def _init_worker(file_sequence, cancel) -> None: global _FILE_SEQUENCE, _CANCEL _FILE_SEQUENCE = file_sequence @@ -242,9 +232,8 @@ def run_multiprocess_repackage( if remote_non_s3: _copy_staged_files(write_prefix, prefix_path) - logger.info('Repackage complete -> %d files %s (%.1fs total)', - n_files, _output_filename(prefix_path, sequence=1, subprefix=writer_subprefix), - time.time() - start) + first_file = generate_warc_filename(prefix_path, sequence=1, writer_subprefix=writer_subprefix, gzip=True) + logger.info('Repackage complete -> %d files %s (%.1fs total)', n_files, first_file, time.time() - start) return result finally: shutil.rmtree(tmp_dir, ignore_errors=True) diff --git a/cdx_toolkit/repackage/writers/series.py b/cdx_toolkit/repackage/writers/series.py index 62fedbf..8537307 100644 --- a/cdx_toolkit/repackage/writers/series.py +++ b/cdx_toolkit/repackage/writers/series.py @@ -26,12 +26,13 @@ def generate_warc_filename( writer_subprefix: Optional[str] = None, gzip: bool = False, ) -> str: - """Generate a WARC file name from a prefix (can be a full path), an optional subprefix - (used to distinguish per-process shards), and a sequence index (file-rotation counter).""" + """The WARC file name [-]-NNN.warc[.gz]: the prefix can be a path or URL (local, s3://, + hf://), the subprefix is the user's --subprefix, the sequence is the file number (shared by all processes + of a run, at least three digits).""" file_name = dest_prefix + '-' if writer_subprefix is not None: file_name += writer_subprefix + '-' - file_name += f'{sequence:03d}.warc' # TODO default warc command uses ".extracted.warc" + file_name += f'{sequence:03d}.warc' if gzip: file_name += '.gz' @@ -40,7 +41,8 @@ def generate_warc_filename( def existing_warc_files(prefix_path: str, writer_subprefix: Optional[str] = None, gzip: bool = True) -> List[str]: """The files of the series [-]-NNN.warc[.gz] (async WARC fetcher) or - [-]-NNNNNN.extracted.warc[.gz] (--wb) that already exist (local, s3://, hf://, ...).""" + [-]-NNNNNN.extracted.warc[.gz] (written by --wb in older versions) that already exist + (local, s3://, hf://, ...).""" fs, fs_prefix = fsspec.url_to_fs(prefix_path) name = re.escape(posixpath.basename(fs_prefix)) + '-' if writer_subprefix is not None: diff --git a/tests/repackage/test_multiprocess_naming.py b/tests/repackage/test_multiprocess_naming.py index a7f61aa..5ac1440 100644 --- a/tests/repackage/test_multiprocess_naming.py +++ b/tests/repackage/test_multiprocess_naming.py @@ -7,18 +7,18 @@ import os from cdx_toolkit.repackage import multiprocess -from cdx_toolkit.repackage.multiprocess import _copy_staged_files, _output_filename - - -def test_output_filename_local_global_series(): - # No subprefix -> the clean global series matching single-process naming. - assert _output_filename('/out/run', 1) == '/out/run-001.warc.gz' - assert _output_filename('/out/run', 12) == '/out/run-012.warc.gz' - assert _output_filename('/out/run', 2, subprefix='SUB') == '/out/run-SUB-002.warc.gz' - - -def test_output_filename_s3(): - assert _output_filename('s3://b/p/run', 3) == 's3://b/p/run-003.warc.gz' +from cdx_toolkit.repackage.multiprocess import _copy_staged_files +from cdx_toolkit.repackage.writers.series import generate_warc_filename + + +def test_warc_filenames(): + # the formatter of the file names, also for the log line of the multi-process mode + assert generate_warc_filename('/out/run', 1, gzip=True) == '/out/run-001.warc.gz' + assert generate_warc_filename('/out/run', 12, gzip=True) == '/out/run-012.warc.gz' + assert generate_warc_filename('/out/run', 1234, gzip=True) == '/out/run-1234.warc.gz' + assert generate_warc_filename('/out/run', 2, writer_subprefix='SUB', gzip=True) == '/out/run-SUB-002.warc.gz' + assert generate_warc_filename('s3://b/p/run', 3, gzip=True) == 's3://b/p/run-003.warc.gz' + assert generate_warc_filename('hf://buckets/a/b/run', 3) == 'hf://buckets/a/b/run-003.warc' def _take_numbers(n): From 35683a27e4d1e748759cda2b87b1fa65baffcd97 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 18:34:45 +0200 Subject: [PATCH 82/91] fix(myrequests): retry when the CDX index server answers with a connection error index.commoncrawl.org answered with the text of an exception ("Error: ('Connection aborted.', RemoteDisconnected(...") when it could not read its index, with a status that was not retried, so the response could not be decoded (seen in the live tests with --closest). Such answers are now retried like a 503; the decode error names the status. --- cdx_toolkit/__init__.py | 2 +- cdx_toolkit/myrequests.py | 19 +++++++++++++++++-- tests/unit/test_myrequests.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/cdx_toolkit/__init__.py b/cdx_toolkit/__init__.py index efa2d9f..896fb5c 100644 --- a/cdx_toolkit/__init__.py +++ b/cdx_toolkit/__init__.py @@ -84,7 +84,7 @@ def cdx_to_captures(resp, wb=None, warc_download_prefix=None): ret = munge_fields(fields, lines) return [CaptureObject(r, wb=wb, warc_download_prefix=warc_download_prefix) for r in ret] - raise ValueError('cannot decode response, first bytes are '+repr(text[:50])) # pragma: no cover + raise ValueError(f'cannot decode response (status {resp.status_code}), first bytes are {text[:50]!r}') class CaptureObject(MutableMapping): diff --git a/cdx_toolkit/myrequests.py b/cdx_toolkit/myrequests.py index f722d14..85487ec 100644 --- a/cdx_toolkit/myrequests.py +++ b/cdx_toolkit/myrequests.py @@ -24,6 +24,20 @@ } +# A CDX index server (pywb) that cannot read its index (e.g. from S3) answers with the text of the exception, e.g. +# "Error: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))", and a +# status that is not retried otherwise: a temporary failure +TEMPORARY_CDX_ERRORS = ( + 'Connection aborted', 'RemoteDisconnected', 'Connection reset', 'Read timed out', 'ConnectTimeout', + 'Max retries exceeded', +) + + +def is_temporary_cdx_error(resp) -> bool: + text = resp.text[:1000] + return text.startswith('Error:') and any(marker in text for marker in TEMPORARY_CDX_ERRORS) + + def dns_fatal(hostname) -> bool: """We have a dns error, should we fail immediately or not? Not for a host that worked before (the error may be temporary).""" @@ -128,7 +142,8 @@ def myrequests_get( try: LOGGER.debug('getting %s %r', url, params) resp = requests.get(url, params=params, headers=headers, timeout=(30.0, 30.0), allow_redirects=False) - if cdx and resp.status_code in {400, 404}: + temporary_cdx_error = cdx and is_temporary_cdx_error(resp) + if cdx and resp.status_code in {400, 404} and not temporary_cdx_error: # 400: ia html error page -- probably page= is too big -- not an error # 404: pywb {'error': 'No Captures found for: www.pbxxxxxxm.com/*'} -- not an error LOGGER.debug('giving up with status %d, no captures found', resp.status_code) @@ -137,7 +152,7 @@ def myrequests_get( if allow404 and resp.status_code == 404: retry = False break - if resp.status_code in {429, 500, 502, 503, 504, 509}: # pragma: no cover + if resp.status_code in {429, 500, 502, 503, 504, 509} or temporary_cdx_error: # 503=slow down, 50[24] are temporary outages, 500=Amazon S3 generic error # CC takes a 503 from storage and then emits a 500 with error text in resp.text # I have never seen IA or CC send 429 or 509, but just in case... diff --git a/tests/unit/test_myrequests.py b/tests/unit/test_myrequests.py index e5d072d..e610674 100644 --- a/tests/unit/test_myrequests.py +++ b/tests/unit/test_myrequests.py @@ -106,3 +106,33 @@ def get(url, **kwargs): starts.sort() gaps = [b - a for a, b in zip(starts, starts[1:])] assert min(gaps) > interval * 0.8, gaps + + +ABORTED = "Error: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))" + + +@pytest.mark.parametrize('status', [200, 400, 500]) +@responses.activate +def test_cdx_server_connection_error_is_retried(monkeypatch, status): + """The index server could not read its index: its answer is the exception's text, with any status.""" + from cdx_toolkit import myrequests + + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'cdx.example.org', 0) + responses.add(responses.GET, 'https://cdx.example.org/cdx', body=ABORTED, status=status) + responses.add(responses.GET, 'https://cdx.example.org/cdx', body='{"url": "https://example.com/"}\n') + + resp = myrequests_get('https://cdx.example.org/cdx', cdx=True) + + assert resp.status_code == 200 and resp.text.startswith('{') + assert len(responses.calls) == 2 + + +@responses.activate +def test_cdx_400_without_connection_error_is_not_retried(monkeypatch): + from cdx_toolkit import myrequests + + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'cdx.example.org', 0) + responses.add(responses.GET, 'https://cdx.example.org/cdx', body='page too big', status=400) + + assert myrequests_get('https://cdx.example.org/cdx', cdx=True).status_code == 400 + assert len(responses.calls) == 1 From 55c7add325f086d912f391865955cbb5101c4475 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 20:07:35 +0200 Subject: [PATCH 83/91] fix(repackage): Ctrl-C on Python < 3.11 and exit with a blocked stdin reader (flaky CI) - Before 3.11, asyncio.run() raises KeyboardInterrupt wherever the main thread is, e.g. after the writer created a file and before it registered it, so abort() missed it (Linux/3.9: 15 of 40 runs left a file, 10 hung). Repackager.run() now cancels the pipeline on SIGINT, like asyncio.run() does since 3.11. - If cdxt stops while the source thread still waits for input (stdin from a process that keeps running), Python can hang at exit (on Windows, closing stdin waits for the pending read): the CLI then writes its message and exits without finalization (os._exit). --- cdx_toolkit/cli/__init__.py | 32 ++++++++++++++++++++--- cdx_toolkit/repackage/repackager.py | 40 ++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py index 4f58d77..a0bbc56 100644 --- a/cdx_toolkit/cli/__init__.py +++ b/cdx_toolkit/cli/__init__.py @@ -1,5 +1,8 @@ from argparse import ArgumentParser +import logging +import os import sys +import traceback from cdx_toolkit.aws import configure_fsspec_s3 from cdx_toolkit.hf import check_huggingface_hub, uses_hf @@ -47,8 +50,31 @@ def main(args=None): try: cmd.func(cmd, cmdline) except (UsageError, FileNotFoundError) as e: - parser.exit(2, f'cdxt {cmd.cmd}: error: {e}\n') + exit_cli(parser, 2, f'cdxt {cmd.cmd}: error: {e}\n') except IncompleteError as e: - parser.exit(1, f'cdxt {cmd.cmd}: error: {e}\n') + exit_cli(parser, 1, f'cdxt {cmd.cmd}: error: {e}\n') except KeyboardInterrupt: - parser.exit(130, f'cdxt {cmd.cmd}: interrupted\n') + exit_cli(parser, 130, f'cdxt {cmd.cmd}: interrupted\n') + except Exception: + if not source_still_reading(): + raise + exit_cli(parser, 1, traceback.format_exc()) + + +def source_still_reading() -> bool: + """Whether cdxt repackage stopped while its source thread still waits for input (e.g. stdin).""" + repackager = sys.modules.get('cdx_toolkit.repackage.repackager') + return repackager is not None and repackager.source_still_reading() + + +def exit_cli(parser: ArgumentParser, status: int, message: str) -> None: + """Exit with the status and message. If a source thread still waits for its input, Python could hang at exit + (on Windows, closing stdin waits for a pending read of another thread): then exit without finalization; the + outputs are complete or removed at this point.""" + if source_still_reading(): + sys.stderr.write(message) + logging.shutdown() + sys.stdout.flush() + sys.stderr.flush() + os._exit(status) + parser.exit(status, message) diff --git a/cdx_toolkit/repackage/repackager.py b/cdx_toolkit/repackage/repackager.py index 8ad15cf..0441bd2 100644 --- a/cdx_toolkit/repackage/repackager.py +++ b/cdx_toolkit/repackage/repackager.py @@ -3,6 +3,8 @@ import contextlib import logging import os +import signal +import sys import threading import time from dataclasses import dataclass, field @@ -21,6 +23,15 @@ logger = logging.getLogger(__name__) +# the name of the thread that reads the source (see Repackager._produce_range_jobs and source_still_reading) +SOURCE_THREAD_NAME = 'cdxt-source' + + +def source_still_reading() -> bool: + """Whether a source thread is still reading its input (e.g. blocked on stdin from a process that keeps + running) after its pipeline stopped.""" + return any(t.name == SOURCE_THREAD_NAME and t.is_alive() for t in threading.enumerate()) + @dataclass class ThroughputTracker: @@ -163,12 +174,38 @@ def run(self) -> RepackageResult: except ImportError: logger.warning('CDXT_UVLOOP=1 but uvloop is not installed; using default asyncio loop') try: - return runner(self.run_async()) + return runner(self._run_with_sigint()) except asyncio.CancelledError: if self.cancel is not None and self.cancel.is_set(): raise RepackageCancelled('stopped (cancel event)') from None raise + async def _run_with_sigint(self) -> RepackageResult: + """run_async(); before Python 3.11, Ctrl-C (SIGINT) cancels it like asyncio.run() does since 3.11, so that it + stops at an await (and aborts the current file) instead of wherever KeyboardInterrupt is raised.""" + if sys.version_info >= (3, 11): + return await self.run_async() + loop = asyncio.get_running_loop() + task = asyncio.current_task() + interrupted = [] + + def on_sigint(): + interrupted.append(True) + task.cancel() + + try: + loop.add_signal_handler(signal.SIGINT, on_sigint) + except (NotImplementedError, RuntimeError, ValueError): # Windows, or not the main thread + return await self.run_async() + try: + return await self.run_async() + except asyncio.CancelledError: + if interrupted: + raise KeyboardInterrupt from None + raise + finally: + loop.remove_signal_handler(signal.SIGINT) + async def _watch_cancel(self, pipeline: asyncio.Task) -> None: """Cancel the pipeline once the cancel event is set.""" while not self.cancel.is_set(): @@ -226,6 +263,7 @@ async def _produce_range_jobs(self, range_jobs_queue: asyncio.Queue, stop: threa loop = asyncio.get_running_loop() def drain() -> int: + threading.current_thread().name = SOURCE_THREAD_NAME count = 0 for job in self.source.iter_range_jobs(): if stop.is_set(): From f3377ddc605525c38109ab78fc41aee68063b256 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 20:22:56 +0200 Subject: [PATCH 84/91] fix(cli): exit without finalization only for the source thread of this run The check looked at every thread named cdxt-source, so a thread left waiting by another run in the same process (in CI: an earlier test) made an unrelated error exit with status 1, or end the whole process with os._exit. The CLI now only considers source threads that were not running before the command. --- cdx_toolkit/cli/__init__.py | 27 ++++++++++++++------------- cdx_toolkit/repackage/repackager.py | 8 ++++---- tests/unit/test_cli.py | 16 ++++++++++++++++ 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py index a0bbc56..f20b6a0 100644 --- a/cdx_toolkit/cli/__init__.py +++ b/cdx_toolkit/cli/__init__.py @@ -47,31 +47,32 @@ def main(args=None): check_huggingface_hub() # before any work, not when the first hf:// path is opened except RuntimeError as e: parser.exit(2, f'cdxt {cmd.cmd}: error: {e}\n') + running = source_threads() # of other runs in this process (e.g. of the Python API) try: cmd.func(cmd, cmdline) except (UsageError, FileNotFoundError) as e: - exit_cli(parser, 2, f'cdxt {cmd.cmd}: error: {e}\n') + exit_cli(parser, 2, f'cdxt {cmd.cmd}: error: {e}\n', running) except IncompleteError as e: - exit_cli(parser, 1, f'cdxt {cmd.cmd}: error: {e}\n') + exit_cli(parser, 1, f'cdxt {cmd.cmd}: error: {e}\n', running) except KeyboardInterrupt: - exit_cli(parser, 130, f'cdxt {cmd.cmd}: interrupted\n') + exit_cli(parser, 130, f'cdxt {cmd.cmd}: interrupted\n', running) except Exception: - if not source_still_reading(): + if not source_threads() - running: raise - exit_cli(parser, 1, traceback.format_exc()) + exit_cli(parser, 1, traceback.format_exc(), running) -def source_still_reading() -> bool: - """Whether cdxt repackage stopped while its source thread still waits for input (e.g. stdin).""" +def source_threads() -> set: + """The running source threads of cdxt repackage (see cdx_toolkit.repackage.repackager.source_threads).""" repackager = sys.modules.get('cdx_toolkit.repackage.repackager') - return repackager is not None and repackager.source_still_reading() + return repackager.source_threads() if repackager is not None else set() -def exit_cli(parser: ArgumentParser, status: int, message: str) -> None: - """Exit with the status and message. If a source thread still waits for its input, Python could hang at exit - (on Windows, closing stdin waits for a pending read of another thread): then exit without finalization; the - outputs are complete or removed at this point.""" - if source_still_reading(): +def exit_cli(parser: ArgumentParser, status: int, message: str, running: set) -> None: + """Exit with the status and message. If the source thread of this run still waits for its input (not one of + the threads `running` before the run), Python could hang at exit (on Windows, closing stdin waits for a pending + read of another thread): then exit without finalization; the outputs are complete or removed at this point.""" + if source_threads() - running: sys.stderr.write(message) logging.shutdown() sys.stdout.flush() diff --git a/cdx_toolkit/repackage/repackager.py b/cdx_toolkit/repackage/repackager.py index 0441bd2..dd8ceeb 100644 --- a/cdx_toolkit/repackage/repackager.py +++ b/cdx_toolkit/repackage/repackager.py @@ -27,10 +27,10 @@ SOURCE_THREAD_NAME = 'cdxt-source' -def source_still_reading() -> bool: - """Whether a source thread is still reading its input (e.g. blocked on stdin from a process that keeps - running) after its pipeline stopped.""" - return any(t.name == SOURCE_THREAD_NAME and t.is_alive() for t in threading.enumerate()) +def source_threads() -> set: + """The source threads that are running: reading their input, maybe blocked on it (e.g. on stdin from a + process that keeps running) after their pipeline stopped.""" + return {t for t in threading.enumerate() if t.name == SOURCE_THREAD_NAME and t.is_alive()} @dataclass diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index c34832f..1e9b51e 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -339,6 +339,22 @@ def test_repackage_fails_if_the_output_cannot_be_created(tmp_path): assert list(tmp_path.glob('out-*')) == [] +def test_a_blocked_source_thread_of_another_run_is_ignored(tmp_path): + """A source thread left waiting by another run in this process (e.g. of the Python API) does not make the + CLI exit without finalization: the error is raised as usual.""" + import threading + + from cdx_toolkit.repackage.repackager import SOURCE_THREAD_NAME + + release = threading.Event() + other = threading.Thread(target=release.wait, name=SOURCE_THREAD_NAME, daemon=True) + other.start() + try: + test_repackage_fails_if_the_output_cannot_be_created(tmp_path) + finally: + release.set() + + def test_repackage_with_wb_counts_failed_captures(tmp_path, capsys): path = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE, IA_CAPTURE]) fetched = [RuntimeError('404'), _make_fake_record()] From b51aaf40cace673d801c252d3e53d66b05507ddc Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 20:41:24 +0200 Subject: [PATCH 85/91] test: the rate limiter test checks the reserved turns, not measured start times On a slow CI runner (macOS) a thread that wakes up late for its turn starts close to the next one; the test now checks that every attempt, also the retry, reserves its own turn, the minimum interval from the others. --- tests/unit/test_myrequests.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_myrequests.py b/tests/unit/test_myrequests.py index e610674..d3f3a50 100644 --- a/tests/unit/test_myrequests.py +++ b/tests/unit/test_myrequests.py @@ -74,10 +74,10 @@ def test_dns_error_is_only_fatal_for_unknown_hosts(monkeypatch, host, fatal): def test_retries_take_their_turn(monkeypatch): - """Three parallel requests to a host, the first answered with a 503: its retry keeps the minimum interval - to the other requests too (it used to start together with the third one).""" + """Three parallel requests to a host, the first answered with a 503: every attempt, also the retry, reserves + its own turn, the minimum interval from the others (the retry used to start in the third request's turn). + The reserved times are checked, not the measured ones, which a slow machine delays.""" import threading - import time from unittest.mock import Mock from cdx_toolkit import myrequests @@ -85,27 +85,36 @@ def test_retries_take_their_turn(monkeypatch): host = 'retry-turn.example.org' interval = 0.1 monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, host, interval) - starts = [] + reserved = [] + original_reserve = myrequests.RATE_LIMITER.reserve + + def reserve(hostname, now): + fetch_time = original_reserve(hostname, now) + reserved.append(fetch_time) + return fetch_time + + monkeypatch.setattr(myrequests.RATE_LIMITER, 'reserve', reserve) + calls = [] lock = threading.Lock() def get(url, **kwargs): with lock: - starts.append(time.monotonic()) - first = len(starts) == 1 + calls.append(url) + first = len(calls) == 1 return Mock(status_code=503 if first else 200, text='') with patch('requests.get', side_effect=get): threads = [threading.Thread(target=myrequests_get, args=(f'https://{host}/{i}',)) for i in range(3)] for thread in threads: thread.start() - time.sleep(0.01) # the first thread takes the first turn for thread in threads: thread.join() - assert len(starts) == 4 - starts.sort() - gaps = [b - a for a, b in zip(starts, starts[1:])] - assert min(gaps) > interval * 0.8, gaps + assert len(calls) == 4 + assert len(reserved) == 4 # one turn per attempt + reserved.sort() + gaps = [b - a for a, b in zip(reserved, reserved[1:])] + assert min(gaps) >= interval - 1e-9, gaps ABORTED = "Error: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))" From 9e96e6745741888bfcdd67266484e8079eb08da9 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 21:08:06 +0200 Subject: [PATCH 86/91] test: the stdin subprocess tests do not pipe stderr without reading it The child's stderr (log lines and a traceback, about 3 KB, more with Windows paths) went to a pipe that was only read after the child exited; a full pipe (a few KB on Windows) blocks the child, so the test timed out on Windows. stderr goes to a file now, and the child dumps the stacks of its threads if it hangs. --- tests/repackage/test_repackager.py | 39 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/tests/repackage/test_repackager.py b/tests/repackage/test_repackager.py index 17b3372..dbc7f84 100644 --- a/tests/repackage/test_repackager.py +++ b/tests/repackage/test_repackager.py @@ -600,8 +600,10 @@ def main(): CHILD = ''' +import faulthandler import sys from unittest.mock import patch +faulthandler.dump_traceback_later(20, exit=True) # the stacks of all threads if it hangs from cdx_toolkit.cli import main from cdx_toolkit.repackage.writers.local import LocalOutputFile from tests.repackage.test_repackager import fake_get @@ -623,21 +625,26 @@ def test_writer_error_with_stdin_still_open(tmp_path): import sys root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - proc = subprocess.Popen( - [sys.executable, '-c', CHILD, str(tmp_path / 'out')], cwd=root, stdin=subprocess.PIPE, - stderr=subprocess.PIPE, text=True, - ) - try: - proc.stdin.write(json.dumps(CAPTURES[0]) + '\n') - proc.stdin.flush() # and stdin stays open - proc.wait(timeout=30) - stderr = proc.stderr.read() - finally: - if proc.poll() is None: - proc.kill() - proc.stdin.close() - assert proc.returncode != 0 - assert 'disk full' in stderr + # stderr to a file: a pipe that is not read while the process runs would block it once full (a few KB on + # Windows) + stderr_path = tmp_path / 'stderr.txt' + with open(stderr_path, 'w') as stderr_file: + proc = subprocess.Popen( + [sys.executable, '-c', CHILD, str(tmp_path / 'out')], cwd=root, stdin=subprocess.PIPE, + stderr=stderr_file, text=True, + ) + try: + proc.stdin.write(json.dumps(CAPTURES[0]) + '\n') + proc.stdin.flush() # and stdin stays open + proc.wait(timeout=30) + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + proc.stdin.close() + stderr = stderr_path.read_text() + assert proc.returncode != 0, stderr + assert 'disk full' in stderr, stderr assert glob.glob(str(tmp_path / 'out-*')) == [] @@ -663,7 +670,7 @@ def test_ctrl_c_with_stdin_still_open(tmp_path): root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) proc = subprocess.Popen( [sys.executable, '-c', CHILD_WITHOUT_FAILURE, str(tmp_path / 'out')], cwd=root, stdin=subprocess.PIPE, - stderr=subprocess.PIPE, text=True, + stderr=subprocess.DEVNULL, text=True, # not a pipe that nobody reads (it would block the process once full) ) try: proc.stdin.write(json.dumps(CAPTURES[0]) + '\n') From c9c68ba14de5af8aad1cace55aee1259542dfcc2 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 15 Oct 2026 21:22:16 +0200 Subject: [PATCH 87/91] fix(cli): remove the exit without finalization for a blocked source thread It did not fix the Windows timeout (the test's full stderr pipe did, see 9e96e67), and finding the thread of the current run by name is racy: a late thread of an earlier run made an in-process CLI call (in CI: pytest) end with os._exit. The Ctrl-C handling for Python < 3.11 stays. --- cdx_toolkit/cli/__init__.py | 33 +++-------------------------- cdx_toolkit/repackage/repackager.py | 10 --------- tests/unit/test_cli.py | 16 -------------- 3 files changed, 3 insertions(+), 56 deletions(-) diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py index f20b6a0..4f58d77 100644 --- a/cdx_toolkit/cli/__init__.py +++ b/cdx_toolkit/cli/__init__.py @@ -1,8 +1,5 @@ from argparse import ArgumentParser -import logging -import os import sys -import traceback from cdx_toolkit.aws import configure_fsspec_s3 from cdx_toolkit.hf import check_huggingface_hub, uses_hf @@ -47,35 +44,11 @@ def main(args=None): check_huggingface_hub() # before any work, not when the first hf:// path is opened except RuntimeError as e: parser.exit(2, f'cdxt {cmd.cmd}: error: {e}\n') - running = source_threads() # of other runs in this process (e.g. of the Python API) try: cmd.func(cmd, cmdline) except (UsageError, FileNotFoundError) as e: - exit_cli(parser, 2, f'cdxt {cmd.cmd}: error: {e}\n', running) + parser.exit(2, f'cdxt {cmd.cmd}: error: {e}\n') except IncompleteError as e: - exit_cli(parser, 1, f'cdxt {cmd.cmd}: error: {e}\n', running) + parser.exit(1, f'cdxt {cmd.cmd}: error: {e}\n') except KeyboardInterrupt: - exit_cli(parser, 130, f'cdxt {cmd.cmd}: interrupted\n', running) - except Exception: - if not source_threads() - running: - raise - exit_cli(parser, 1, traceback.format_exc(), running) - - -def source_threads() -> set: - """The running source threads of cdxt repackage (see cdx_toolkit.repackage.repackager.source_threads).""" - repackager = sys.modules.get('cdx_toolkit.repackage.repackager') - return repackager.source_threads() if repackager is not None else set() - - -def exit_cli(parser: ArgumentParser, status: int, message: str, running: set) -> None: - """Exit with the status and message. If the source thread of this run still waits for its input (not one of - the threads `running` before the run), Python could hang at exit (on Windows, closing stdin waits for a pending - read of another thread): then exit without finalization; the outputs are complete or removed at this point.""" - if source_threads() - running: - sys.stderr.write(message) - logging.shutdown() - sys.stdout.flush() - sys.stderr.flush() - os._exit(status) - parser.exit(status, message) + parser.exit(130, f'cdxt {cmd.cmd}: interrupted\n') diff --git a/cdx_toolkit/repackage/repackager.py b/cdx_toolkit/repackage/repackager.py index dd8ceeb..6390e7c 100644 --- a/cdx_toolkit/repackage/repackager.py +++ b/cdx_toolkit/repackage/repackager.py @@ -23,15 +23,6 @@ logger = logging.getLogger(__name__) -# the name of the thread that reads the source (see Repackager._produce_range_jobs and source_still_reading) -SOURCE_THREAD_NAME = 'cdxt-source' - - -def source_threads() -> set: - """The source threads that are running: reading their input, maybe blocked on it (e.g. on stdin from a - process that keeps running) after their pipeline stopped.""" - return {t for t in threading.enumerate() if t.name == SOURCE_THREAD_NAME and t.is_alive()} - @dataclass class ThroughputTracker: @@ -263,7 +254,6 @@ async def _produce_range_jobs(self, range_jobs_queue: asyncio.Queue, stop: threa loop = asyncio.get_running_loop() def drain() -> int: - threading.current_thread().name = SOURCE_THREAD_NAME count = 0 for job in self.source.iter_range_jobs(): if stop.is_set(): diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 1e9b51e..c34832f 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -339,22 +339,6 @@ def test_repackage_fails_if_the_output_cannot_be_created(tmp_path): assert list(tmp_path.glob('out-*')) == [] -def test_a_blocked_source_thread_of_another_run_is_ignored(tmp_path): - """A source thread left waiting by another run in this process (e.g. of the Python API) does not make the - CLI exit without finalization: the error is raised as usual.""" - import threading - - from cdx_toolkit.repackage.repackager import SOURCE_THREAD_NAME - - release = threading.Event() - other = threading.Thread(target=release.wait, name=SOURCE_THREAD_NAME, daemon=True) - other.start() - try: - test_repackage_fails_if_the_output_cannot_be_created(tmp_path) - finally: - release.set() - - def test_repackage_with_wb_counts_failed_captures(tmp_path, capsys): path = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE, IA_CAPTURE]) fetched = [RuntimeError('404'), _make_fake_record()] From e2ff0d77ed729c047c69fd6bf17f851b15b03e86 Mon Sep 17 00:00:00 2001 From: malteos Date: Fri, 16 Oct 2026 12:00:10 +0200 Subject: [PATCH 88/91] Keep the 0.9.39 cdxt command lines working (deprecated) (#80) --- CHANGELOG.md | 7 +- README.md | 10 +- cdx_toolkit/cli/__init__.py | 15 +- cdx_toolkit/cli/deprecated.py | 104 ------ cdx_toolkit/cli/legacy.py | 349 ++++++++++++++++++ docs/cli-migration-guide.md | 40 +- .../test_cli/test_iter_alias.jsonl | 1 - tests/test_cli.py | 33 -- tests/test_cli_legacy.py | 192 ++++++++++ tests/unit/test_cli.py | 46 --- tests/unit/test_cli_legacy.py | 286 ++++++++++++++ 11 files changed, 867 insertions(+), 216 deletions(-) delete mode 100644 cdx_toolkit/cli/deprecated.py create mode 100644 cdx_toolkit/cli/legacy.py delete mode 100644 tests/data/mock_responses/test_cli/test_iter_alias.jsonl create mode 100644 tests/test_cli_legacy.py create mode 100644 tests/unit/test_cli_legacy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bb0783..13e97a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ - unreleased + new command line interface: `cdxt index` and `cdxt repackage` (see README) - + `cdxt iter` is a deprecated alias of `cdxt index` (same arguments, prints a deprecation warning) - + removed `cdxt size` and `cdxt warc`; they print the equivalent new command + + the command lines of 0.9.39 (`cdxt --cc iter URL`, `size`, `warc`) are deprecated (to be removed in a future version): they run + with the new code and print a suggested new command line; `iter` output, `--get` and `size --details` are kept; + `warc` writes `TEST-001.warc.gz` (was `TEST-000000.extracted.warc.gz`) with the records copied byte-for-byte + (no `WARC-Source-URI`/`WARC-Source-Range` headers), no file without captures, and exits with status 1 if + captures could not be fetched; `iter --all-fields --csv` is a usage error (was a traceback) + `cdxt index`: `--hostname`, `--domain`, repeatable `--url`, `-o/--output` (local or remote), JSONL/CDXJ/CSV/TSV output, all fields by default, `--count` replaces `size` + `cdxt repackage`: reads captures from a file or stdin, human-readable `--size`, `--is-part-of`, `--description` diff --git a/README.md b/README.md index 6ac7ea1..ea88ee0 100644 --- a/README.md +++ b/README.md @@ -317,10 +317,12 @@ upload (an S3 lifecycle rule for incomplete uploads removes it). ### Migrating from the old commands -`cdxt iter` is a deprecated alias of `cdxt index` with the same (new) -arguments; it prints a deprecation warning. The `size` and `warc` commands -have been removed; running them prints the equivalent new command. See the [migration guide](docs/cli-migration-guide.md) -for all changes and more examples. +The command lines of cdx_toolkit 0.9.39 (`cdxt --cc iter ...`, `size`, +`warc`) still work, but are deprecated and will be removed in a future +version; they print a suggested new command line. The old `warc` now writes +`TEST-001.warc.gz` and exits with status 1 if captures could not be fetched. +See the [migration guide](docs/cli-migration-guide.md) for all changes and +more examples. | Old | New | |---|---| diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py index 4f58d77..c4e8aef 100644 --- a/cdx_toolkit/cli/__init__.py +++ b/cdx_toolkit/cli/__init__.py @@ -8,8 +8,7 @@ from . import index, repackage from .common import IncompleteError, add_verbose_arg, set_loglevel -from .deprecated import DEPRECATED_COMMANDS, REMOVED_COMMANDS, exit_removed_command, get_command -from .deprecated import rewrite_deprecated_command +from .legacy import build_legacy_parser, deprecation_message, is_legacy_command_line def build_parser() -> ArgumentParser: @@ -28,15 +27,13 @@ def main(args=None): if args is None: # pragma: no cover args = sys.argv[1:] - command = get_command(args) - if command in REMOVED_COMMANDS: - exit_removed_command(command, args) - if command in DEPRECATED_COMMANDS: - args = rewrite_deprecated_command(args) - - parser = build_parser() + # the command lines of cdx_toolkit 0.9.39 (deprecated): `cdxt --cc iter URL`, `size`, `warc` + legacy = is_legacy_command_line(args) + parser = build_legacy_parser() if legacy else build_parser() cmd = parser.parse_args(args=args) set_loglevel(cmd.verbose) + if legacy: + sys.stderr.write(deprecation_message(cmd)) configure_fsspec_s3() # Common Crawl's S3 defaults for s3:// paths, unless configured by the user cmdline = ' '.join(['cdxt'] + list(args)) if uses_hf(args): diff --git a/cdx_toolkit/cli/deprecated.py b/cdx_toolkit/cli/deprecated.py deleted file mode 100644 index fed3d28..0000000 --- a/cdx_toolkit/cli/deprecated.py +++ /dev/null @@ -1,104 +0,0 @@ -''' -Old commands: `iter` is a deprecated alias of `index` (same arguments); `size` and `warc` were removed and -print the equivalent new command line. -''' -import shlex -import sys -from typing import List, Optional, Tuple - - -DEPRECATED_COMMANDS = {'iter': 'index'} -REMOVED_COMMANDS = {'size': 'index --count', 'warc': 'index | cdxt repackage'} - - -DOCS_URL = 'https://github.com/commoncrawl/cdx_toolkit#command-line-tools' - -# options of the old command line that take a value -VALUE_OPTIONS = { - '--crawl', '--source', '--wb', '--limit', '--cc-mirror', '--cc-sort', '--from', '--to', '--filter', - '--closest', '--fields', '--prefix', '--subprefix', '--size', '--creator', '--operator', '--url-fgrep', - '--url-fgrepv', '--warc-download-prefix', -} -INDEX_OPTIONS = { - '--crawl', '--limit', '--cc-mirror', '--cc-sort', '--from', '--to', '--filter', '--closest', '--fields', - '--url-fgrep', '--url-fgrepv', -} -REPACKAGE_OPTIONS = {'--wb', '--prefix', '--subprefix', '--size', '--creator', '--operator', '--warc-download-prefix'} - - -def parse_old_args(args: List[str]) -> Tuple[List[Tuple[str, Optional[str]]], List[Tuple[int, str]]]: - """Split a command line into (option, value) pairs and (position, value) of the positional arguments.""" - options, positionals = [], [] - tokens = iter(enumerate(args)) - for i, token in tokens: - if token.startswith('-') and token != '-': - if '=' in token: - options.append(tuple(token.split('=', 1))) - elif token in VALUE_OPTIONS: - options.append((token, next(tokens, (None, ''))[1])) - else: - options.append((token, None)) - else: - positionals.append((i, token)) - return options, positionals - - -def get_command(args: List[str]) -> Optional[str]: - _, positionals = parse_old_args(args) - return positionals[0][1] if positionals else None - - -def translate(args: List[str]) -> str: - """Best-effort translation of an old size/warc command line to the new syntax.""" - options, positionals = parse_old_args(args) - command, urls = positionals[0][1], [url for _, url in positionals[1:]] - - index, index_args, repackage_args = None, [], [] - for option, value in options: - if option in ('--cc', '--ia'): - index = option[2:] - elif option == '--source': - index = value - elif option in ('-v', '-vv', '--verbose'): - index_args.append(option) - elif option in INDEX_OPTIONS: - index_args += [option, value] - elif option in REPACKAGE_OPTIONS: - repackage_args += [option, value] - # --get, --all-fields and --details have no equivalent (all fields are the default) - - if index is None: - index = 'cc' if any(option == '--crawl' for option, _ in options) else 'INDEX' - url_args = [arg for url in urls for arg in ('--url', url)] - - line = ['cdxt', 'index', index] + url_args + index_args - if command == 'size': - line.append('--count') - elif command == 'warc': - if index == 'ia' and '--wb' not in repackage_args: - repackage_args += ['--wb', 'ia'] - line += ['|', 'cdxt', 'repackage'] + repackage_args - return ' '.join(token if token == '|' else shlex.quote(token) for token in line) - - -def rewrite_deprecated_command(args: List[str]) -> List[str]: - """Run `cdxt iter ...` exactly like `cdxt index ...` (same arguments), with a deprecation warning on stderr.""" - _, positionals = parse_old_args(args) - position, command = positionals[0] - new_command = DEPRECATED_COMMANDS[command] - sys.stderr.write( - f'cdxt: "{command}" is deprecated and will be removed in a future version, use "cdxt {new_command}" ' - f'(same arguments). See {DOCS_URL}\n' - ) - return args[:position] + [new_command] + args[position + 1:] - - -def exit_removed_command(command: str, args: List[str]): - message = f'cdxt: the "{command}" command has been removed, use "cdxt {REMOVED_COMMANDS[command]}" instead.\n' - try: - message += f'\nThe equivalent command is:\n\n {translate(args)}\n' - except Exception: # pragma: no cover - the translation is only a hint - pass - message += f'\nSee "cdxt index --help", "cdxt repackage --help" or {DOCS_URL}\n' - sys.stderr.write(message) - raise SystemExit(2) diff --git a/cdx_toolkit/cli/legacy.py b/cdx_toolkit/cli/legacy.py new file mode 100644 index 0000000..8596236 --- /dev/null +++ b/cdx_toolkit/cli/legacy.py @@ -0,0 +1,349 @@ +''' +The command lines of cdx_toolkit 0.9.39 and earlier, e.g. `cdxt --cc --limit 10 iter 'example.com/*'`: options before +the command, and the commands `iter`, `size` and `warc`. They are deprecated: they are parsed with the old grammar and +run with the current code (CDX API backend, repackage()), printing a suggested new command line on stderr. + +Differences to 0.9.39: `warc` writes [-]-001.warc.gz (was -000000.extracted.warc.gz), copies the +WARC records byte-for-byte, writes no file without captures, and exits with status 1 if a capture could not be +fetched; `iter --all-fields --csv` is a usage error (it failed with a traceback). +''' +from argparse import ArgumentParser, Namespace +import csv +import json +import logging +import re +import shlex +import sys +from typing import Iterable, List, Mapping + +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index +from cdx_toolkit.indexes.base import filter_by_url +from cdx_toolkit.utils import get_version + +from .common import IncompleteError + + +LOGGER = logging.getLogger(__name__) + +LEGACY_COMMANDS = ('iter', 'size', 'warc') + +DOCS_URL = 'https://github.com/commoncrawl/cdx_toolkit#command-line-tools' + +# options of the old command line (before the command) that take a value +GLOBAL_VALUE_OPTIONS = { + '--crawl', '--source', '--wb', '--limit', '--cc-mirror', '--cc-sort', '--from', '--to', '--filter', '--closest', +} + +DEFAULT_FIELDS = 'url,status,timestamp' + + +def is_new_top_level_option(token: str) -> bool: + """The options of the new command line before the command: -h, -V and -v.""" + return token in ('-h', '--help', '-V', '--version', '--verbose') or re.fullmatch(r'-v+', token) is not None + + +def is_legacy_command_line(args: List[str]) -> bool: + """A command line of cdx_toolkit 0.9.39: the command is iter, size or warc, or options other than -h, -V and -v + come before the command (the new command line has all other options after the command). The values of the old + options are skipped, so `cdxt --source iter ...` is not taken for the command `iter`.""" + before_command = [] + command = None + tokens = iter(args) + for token in tokens: + if token == '--': + command = next(tokens, None) + break + if token.startswith('-') and token != '-': + before_command.append(token) + if '=' not in token and token in GLOBAL_VALUE_OPTIONS: + next(tokens, None) + continue + command = token + break + return command in LEGACY_COMMANDS or not all(is_new_top_level_option(token) for token in before_command) + + +def build_legacy_parser() -> ArgumentParser: + """The command line of cdx_toolkit 0.9.39.""" + parser = ArgumentParser(prog='cdxt', description='cdx_toolkit iterator command line tool (deprecated syntax)') + + parser.add_argument('--version', '-V', action='version', version=get_version()) + parser.add_argument('--verbose', '-v', action='count', help='set logging level to INFO (-v) or DEBUG (-vv)') + parser.add_argument('--cc', action='store_const', const='cc', help='direct the query to the Common Crawl CDX/WARCs') + parser.add_argument( + '--crawl', + action='store', + help='crawl names (comma separated) or an integer for the most recent N crawls. Implies --cc', + ) + parser.add_argument( + '--ia', action='store_const', const='ia', help='direct the query to the Internet Archive CDX/wayback' + ) + parser.add_argument('--source', action='store', help='direct the query to this CDX server') + parser.add_argument('--wb', action='store', help='direct replays for content to this wayback') + parser.add_argument('--limit', type=int, action='store') + parser.add_argument('--cc-mirror', action='store', help='use this Common Crawl index mirror') + parser.add_argument('--cc-sort', action='store', help='default mixed, alternatively: ascending') + parser.add_argument('--from', action='store') + parser.add_argument('--to', action='store') + parser.add_argument('--filter', action='append', help='see CDX API documentation for usage') + parser.add_argument( + '--get', action='store_true', help='use a single get instead of a paged iteration. default limit=1000' + ) + parser.add_argument( + '--closest', action='store', help='get the closest capture to this timestamp. use with --get' + ) + + subparsers = parser.add_subparsers(dest='cmd') + subparsers.required = True + + iterate = subparsers.add_parser('iter', help='iterate printing captures') + iterate.add_argument('--all-fields', action='store_true') + iterate.add_argument( + '--fields', action='store', default=DEFAULT_FIELDS, help='try --all-fields if you need the list' + ) + iterate.add_argument('--jsonl', action='store_true') + iterate.add_argument('--csv', action='store_true') + iterate.add_argument('url') + iterate.set_defaults(func=run_iter) + + warc = subparsers.add_parser('warc', help='iterate over capture content, creating a warc') + warc.add_argument('--prefix', default='TEST', help='prefix for the warc filename') + warc.add_argument('--subprefix', type=str, default=None, help='subprefix for the warc filename, default None') + warc.add_argument('--size', type=int, default=1000000000, help='target for the warc filesize in bytes') + warc.add_argument('--creator', action='store', help='creator of the warc: person, organization, service') + warc.add_argument('--operator', action='store', help='a person, if the creator is an organization') + warc.add_argument('--url-fgrep', action='store', help='this pattern must be present to warc an url') + warc.add_argument( + '--url-fgrepv', action='store', help='this pattern must not be present to warc an url, e.g. /robots.txt' + ) + warc.add_argument( + '--warc-download-prefix', action='store', help='prefix for downloading content, automatically set for CC' + ) + warc.add_argument('url') + warc.set_defaults(func=run_warc) + + size = subparsers.add_parser('size', help='imprecise count of how many results are available') + size.add_argument('--details', action='store_true', help='show details of each subindex') + size.add_argument('url') + size.set_defaults(func=run_size) + + return parser + + +def legacy_source(ns: Namespace) -> str: + """The CDX API of the old options: --crawl implies cc, otherwise --cc, --ia, then --source.""" + source = 'cc' if ns.crawl else ns.cc or ns.ia or ns.source + if source is None: + raise UsageError('must specify --cc, --ia, or a --source') + if source not in ('cc', 'ia') and not source.startswith(('http://', 'https://')): + raise UsageError(f'--source {source}: use the http(s) URL of a CDX API server') + return source + + +def request_kwargs(ns: Namespace) -> dict: + """The parameters of the CDX API requests (CDXFetcher.iter/get/get_size_estimate), as in 0.9.39.""" + kwargs = {} + if ns.limit: + kwargs['limit'] = ns.limit + if getattr(ns, 'from'): + kwargs['from_ts'] = getattr(ns, 'from') + if ns.to: + kwargs['to'] = ns.to + if ns.closest: + kwargs['closest'] = ns.closest + if ns.filter: + kwargs['filter'] = ns.filter + return kwargs + + +def cc_options(ns: Namespace, source: str) -> dict: + """--cc-mirror and --cc-sort; the old command line ignored them for other sources.""" + if source != 'cc': + return {} + return {'cc_mirror': ns.cc_mirror, 'cc_sort': ns.cc_sort} + + +def fetcher(ns: Namespace, source: str): + """A CDXFetcher for the requests that the index backend does not make as 0.9.39 did (--get, --closest, size).""" + import cdx_toolkit + from cdx_toolkit.commoncrawl import normalize_crawl + + kwargs = {'source': source} + for name, value in cc_options(ns, source).items(): + if value: + kwargs[name] = value + if ns.crawl: + kwargs['crawl'] = normalize_crawl([ns.crawl]) + return cdx_toolkit.CDXFetcher(**kwargs) + + +def legacy_captures(ns: Namespace, source: str, use_get: bool) -> Iterable[Mapping]: + """The captures of the old command line: a single (non-paged) request with --get, the paged iteration with the + closest= parameter for --closest without --get, otherwise the CDX API backend (the same requests).""" + if use_get: + return fetcher(ns, source).get(ns.url, **request_kwargs(ns)) + if ns.closest: + return fetcher(ns, source).iter(ns.url, **request_kwargs(ns)) + backend = open_index(source, 'cdx-api', **cc_options(ns, source)) + query = IndexQuery( + urls=[ns.url], + crawl=ns.crawl, + from_ts=getattr(ns, 'from'), + to=ns.to, + filters=list(ns.filter or []), + limit=ns.limit or None, + ) + return backend.records(query) + + +def iter_fields(ns: Namespace) -> List[str]: + """The fields of `iter --fields` (default url,status,timestamp): without duplicates, sorted.""" + return sorted(set(ns.fields.split(','))) + + +def run_iter(ns: Namespace, cmdline: str): + """`cdxt ... iter URL`: print the captures as in 0.9.39, as text (`key value, ...`), JSON lines (--jsonl) or + CSV (--csv), with the --fields (sorted), or all fields with --all-fields.""" + if ns.all_fields and ns.csv: + # 0.9.39 wrote the header of --fields, then failed with a traceback at the first capture + raise UsageError('--all-fields cannot be used with --csv, select the columns with --fields') + source = legacy_source(ns) + fields = iter_fields(ns) + + writer = None + if ns.csv: + writer = csv.DictWriter(sys.stdout, fieldnames=fields) + writer.writeheader() + + for capture in legacy_captures(ns, source, use_get=ns.get): + if ns.all_fields: + selected = dict(capture) + else: + selected = {k: capture[k] for k in fields if k in capture} + if ns.jsonl: + print(json.dumps(selected, sort_keys=True)) + elif writer: + writer.writerow(selected) + else: + print(', '.join(f'{k} {selected[k]}' for k in sorted(selected))) + + +def run_size(ns: Namespace, cmdline: str): + """`cdxt ... size URL`: print the estimated number of captures (CDXFetcher.get_size_estimate(), not capped by + --limit), with --details first the estimate of each index.""" + kwargs = request_kwargs(ns) + if ns.details: + kwargs['details'] = True # get_size_estimate() prints the estimate of each index + print(fetcher(ns, legacy_source(ns)).get_size_estimate(ns.url, **kwargs)) + + +def warc_content_source(ns: Namespace, source: str) -> dict: + """Where `warc` gets the content of the captures from (as CDXFetcher did): {'wb': ...} or + {'warc_download_prefix': ...}.""" + if ns.wb and ns.warc_download_prefix: + raise UsageError('cannot specify both --wb and --warc-download-prefix') + if ns.wb: + if source == 'cc': + raise UsageError('cannot specify --wb for Common Crawl (--cc, --crawl)') + return {'wb': ns.wb} + if ns.warc_download_prefix: + return {'warc_download_prefix': ns.warc_download_prefix} + if source == 'cc': + return {'warc_download_prefix': 'https://data.commoncrawl.org'} + if source == 'ia': + return {'wb': 'ia'} + raise UsageError('warc with --source needs --wb (a wayback machine) or --warc-download-prefix') + + +def run_warc(ns: Namespace, cmdline: str): + """`cdxt ... warc URL`: write the captures to [-]-001.warc.gz, ... with repackage(), streamed + in index order; IncompleteError (exit status 1) if captures could not be fetched.""" + from cdx_toolkit.repackage import repackage + + source = legacy_source(ns) + content = warc_content_source(ns, source) + # 0.9.39 iterated (paged) also with --get; the limit applies before --url-fgrep/--url-fgrepv + captures = filter_by_url(legacy_captures(ns, source, use_get=False), ns.url_fgrep, ns.url_fgrepv) + result = repackage( + captures, + ns.prefix, + subprefix=ns.subprefix, + size=ns.size, + creator=ns.creator, + operator=ns.operator, + description='warc extraction generated with: ' + cmdline, + sort=False, # stream the captures in index order, as 0.9.39 did + **content, + ) + LOGGER.info('wrote %d of %d captures to WARC files (%d failed)', result.written, result.requested, result.failed) + if result.failed: + raise IncompleteError( + f'{result.failed} of {result.requested} captures could not be fetched (see the log), ' + f'{result.written} were written' + ) + + +def suggested_replacement(ns: Namespace) -> str: + """The new command line that does (about) the same; not exact for the text output of iter, --get, --closest + without --get and size --details.""" + source = 'cc' if ns.crawl else ns.cc or ns.ia or ns.source or 'INDEX' + index = ['cdxt', 'index', source, '--url', ns.url] + if ns.crawl: + index += ['--crawl', ns.crawl] + if ns.limit: + index += ['--limit', str(ns.limit)] + elif ns.cmd == 'iter' and ns.get: + index += ['--limit', '1000'] # the default limit of a single get + for option, value in (('--from', getattr(ns, 'from')), ('--to', ns.to), ('--closest', ns.closest)): + if value: + index += [option, value] + for expression in ns.filter or []: + index += ['--filter', expression] + if source == 'cc': + for option, value in (('--cc-mirror', ns.cc_mirror), ('--cc-sort', ns.cc_sort)): + if value: + index += [option, value] + + if ns.cmd == 'iter': + fields = [] if ns.all_fields else ['--fields', ','.join(iter_fields(ns))] + if ns.jsonl: + index += fields + else: # CSV instead of the old text output + index += ['--output-format', 'csv'] + fields + return render(index) + if ns.cmd == 'size': + return render(index + ['--count']) + + for option, value in (('--url-fgrep', ns.url_fgrep), ('--url-fgrepv', ns.url_fgrepv)): + if value: + index += [option, value] + repackage = ['cdxt', 'repackage', '--prefix', ns.prefix] + if ns.subprefix: + repackage += ['--subprefix', ns.subprefix] + if ns.size != 1000000000: + repackage += ['--size', str(ns.size)] + for option, value in (('--creator', ns.creator), ('--operator', ns.operator)): + if value: + repackage += [option, value] + try: + content = warc_content_source(ns, legacy_source(ns)) + except UsageError: + content = {} + if 'wb' in content: + repackage += ['--wb', content['wb']] + elif content.get('warc_download_prefix', 'https://data.commoncrawl.org') != 'https://data.commoncrawl.org': + repackage += ['--warc-download-prefix', content['warc_download_prefix']] + return render(index) + ' | ' + render(repackage + ['--no-sort']) + + +def render(tokens: List[str]) -> str: + """A command line from its tokens, quoted for the shell.""" + return ' '.join(shlex.quote(token) for token in tokens) + + +def deprecation_message(ns: Namespace) -> str: + """The line printed on stderr for an old command line, with the suggested new command line.""" + return ( + 'cdxt: this command line uses the old syntax, which is deprecated and will be removed in a future version; ' + f'suggested replacement: {suggested_replacement(ns)} (see {DOCS_URL})\n' + ) diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index a5229a2..1101045 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -7,29 +7,34 @@ The `cdxt` command line tool has a new design with two commands: - **`cdxt repackage`** fetches the captures listed in an index file and writes them into WARC files (replaces `cdxt warc`, together with `cdxt index`). -`cdxt iter` is now a deprecated alias of `cdxt index`: it takes exactly the same (new) arguments, -works the same and prints a deprecation warning. The old `iter` command lines (options before the -command, the URL as a positional argument) no longer work, rewrite them as shown below. +The command lines of cdx_toolkit 0.9.39 and earlier (options before the command, the URL as a +positional argument, the `iter`, `size` and `warc` commands) still work, but they are deprecated and +will be removed in a future version. They run with the new code and print a suggested new command line +on stderr: ``` -$ cdxt iter cc --url 'commoncrawl.org/*' --limit 10 -cdxt: "iter" is deprecated and will be removed in a future version, use "cdxt index" (same arguments). ... -{"digest": ..., "url": "https://commoncrawl.org/", ...} +$ cdxt --cc --limit 10 iter 'commoncrawl.org/*' +cdxt: this command line uses the old syntax, which is deprecated and will be removed in a future version; suggested replacement: cdxt index cc --url 'commoncrawl.org/*' --limit 10 --output-format csv --fields status,timestamp,url (see https://github.com/commoncrawl/cdx_toolkit#command-line-tools) +status 200, timestamp 20250804103329, url https://www.commoncrawl.org/ +... ``` -The `size` and `warc` commands have been removed. If you run one of them, `cdxt` exits with an -error and prints the equivalent new command: +The output of the old `iter` (text, `--jsonl`, `--csv`, the default fields `url,status,timestamp`), +`--get` and `size --details` are kept. The suggested replacement is not always exact: the new +commands have no text output (CSV is suggested instead), `--closest` always makes a single query, and +`cdxt index --count` is at most `--limit` and has no `--details`. -``` -$ cdxt --cc --limit 10 warc 'commoncrawl.org/*' -cdxt: the "warc" command has been removed, use "cdxt index | cdxt repackage" instead. - -The equivalent command is: +The old `warc` command now runs like `cdxt index ... | cdxt repackage --no-sort`, which changes: - cdxt index cc --url 'commoncrawl.org/*' --limit 10 | cdxt repackage +- the WARC files are named `TEST-001.warc.gz`, `TEST-002.warc.gz`, ... (with `--prefix`/`--subprefix`: + `PREFIX-SUBPREFIX-001.warc.gz`) instead of `TEST-000000.extracted.warc.gz`; WARC files with the same + prefix must not exist yet +- the WARC records are copied byte-for-byte, without the added `WARC-Source-URI` and + `WARC-Source-Range` headers +- no WARC file is written without captures +- the exit status is 1 if captures could not be fetched (the others are written) -See "cdxt index --help", "cdxt repackage --help" or https://github.com/commoncrawl/cdx_toolkit#command-line-tools -``` +`cdxt --cc iter --all-fields --csv` is a usage error (it failed with a traceback before). ## Quick reference @@ -130,7 +135,8 @@ cdxt --cc size 'commoncrawl.org/*' cdxt index cc --url 'commoncrawl.org/*' --count ``` -`size --details` (one line per index) has been removed. Use `-v` to see which indexes are queried. +`index --count` has no `--details` (one line per index), and is at most `--limit`. Use `-v` to see which +indexes are queried; the deprecated `cdxt --cc size --details URL` still prints the lines. ### `warc` is now `index` + `repackage` diff --git a/tests/data/mock_responses/test_cli/test_iter_alias.jsonl b/tests/data/mock_responses/test_cli/test_iter_alias.jsonl deleted file mode 100644 index 10fa19f..0000000 --- a/tests/data/mock_responses/test_cli/test_iter_alias.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"method": "GET", "url": "https://index.commoncrawl.org/CC-MAIN-2025-33-index", "request_params": {"limit": 10, "url": "commoncrawl.org/*", "output": "json", "page": 0, "from": "20240915202343"}, "response_status_code": 200, "response_text": "{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804103329\", \"url\": \"https://www.commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"FAPYCDZSWJIQ2YPQAYDRLAC25YMUNVTD\", \"length\": \"7063\", \"offset\": \"667474763\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279781.29/warc/CC-MAIN-20250804083340-20250804113340-00340.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804145408\", \"url\": \"http://commoncrawl.org\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"570\", \"offset\": \"71720\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279808.15/robotstxt/CC-MAIN-20250804145214-20250804175214-00905.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804145409\", \"url\": \"http://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"571\", \"offset\": \"138028\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279808.15/crawldiagnostics/CC-MAIN-20250804145214-20250804175214-00494.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804145409\", \"url\": \"https://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"GWWHOAM46QU5UTPSJAUTVADAM56EUVPS\", \"length\": \"7055\", \"offset\": \"128243158\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279808.15/warc/CC-MAIN-20250804145214-20250804175214-00389.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804180633\", \"url\": \"http://commoncrawl.org\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"568\", \"offset\": \"69067\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279827.21/robotstxt/CC-MAIN-20250804180035-20250804210035-00905.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804180633\", \"url\": \"http://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"568\", \"offset\": \"213023\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279827.21/crawldiagnostics/CC-MAIN-20250804180035-20250804210035-00494.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804180633\", \"url\": \"https://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"UQZZ7W2BXQXNZGXL323YV3RKVIVVYTLL\", \"length\": \"6977\", \"offset\": \"144661303\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279827.21/warc/CC-MAIN-20250804180035-20250804210035-00389.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250805042626\", \"url\": \"http://commoncrawl.org\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"570\", \"offset\": \"60915\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279847.20/robotstxt/CC-MAIN-20250805032822-20250805062822-00905.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250805042627\", \"url\": \"http://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"571\", \"offset\": \"173482\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279847.20/crawldiagnostics/CC-MAIN-20250805032822-20250805062822-00494.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250805042627\", \"url\": \"https://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"UQZZ7W2BXQXNZGXL323YV3RKVIVVYTLL\", \"length\": \"6986\", \"offset\": \"134133724\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279847.20/warc/CC-MAIN-20250805032822-20250805062822-00389.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n", "response_headers": {"Server": "nginx/1.29.1", "Date": "Mon, 15 Sep 2025 20:23:44 GMT", "Content-Type": "text/x-ndjson", "Transfer-Encoding": "chunked", "Connection": "keep-alive", "Content-Encoding": "gzip"}} diff --git a/tests/test_cli.py b/tests/test_cli.py index 690b12c..a0addbb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -310,39 +310,6 @@ def test_multi_misc_ia(capsys, caplog): multi_helper(testdata, capsys, caplog) -@conditional_mock_responses -def test_iter_alias(capsys): - # same requests as test_basics (`index cc --limit 10 --url commoncrawl.org/*`) - main(args='iter cc --limit 10 --url commoncrawl.org/*'.split()) - out, err = capsys.readouterr() - - assert len(out.splitlines()) == 10 - assert all(json.loads(line)['url'] for line in out.splitlines()) - assert '"iter" is deprecated' in err and 'use "cdxt index"' in err - - -@pytest.mark.parametrize( - 'args, expected', - [ - ('--crawl 2 size --details commoncrawl.org/*', "cdxt index cc --url 'commoncrawl.org/*' --crawl 2 --count"), - ( - '-v --ia --limit 1 warc --prefix FOO --url-fgrepv robots.txt commoncrawl.org/*', - "cdxt index ia --url 'commoncrawl.org/*' -v --limit 1 --url-fgrepv robots.txt" - " | cdxt repackage --prefix FOO --wb ia", - ), - ], -) -def test_removed_commands(capsys, args, expected): - with pytest.raises(SystemExit) as e: - main(args=args.split()) - assert e.value.code == 2 - - out, err = capsys.readouterr() - assert out == '' - assert 'has been removed' in err - assert expected in err - - def warc_test_helper(tmpdir, index_args: str, repackage_args: str = ''): with tmpdir.as_cwd(): for cmdline in ( diff --git a/tests/test_cli_legacy.py b/tests/test_cli_legacy.py new file mode 100644 index 0000000..8a929d2 --- /dev/null +++ b/tests/test_cli_legacy.py @@ -0,0 +1,192 @@ +'''The command lines of cdx_toolkit 0.9.39 (deprecated), with the responses recorded for them: the mock data of +tests/test_cli.py was recorded with these command lines (`--cc --limit 10 iter URL` makes the same requests as +`index cc --limit 10 --url URL`).''' +import functools +import json +import re + +import pytest +import responses +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main + +from tests.conftest import LIVE_TESTS, conditional_mock_responses, mock_response_from_jsonl +from tests.test_cli import slow_ci + + +DEPRECATED = 'this command line uses the old syntax, which is deprecated' +TEXT_LINE = re.compile(r'status \d+, timestamp \d{14}, url \S+') + + +def recorded(name): + """The recorded responses of tests/test_cli.py:: (unless CDXT_LIVE_TESTS has http).""" + def decorator(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + if 'http' not in LIVE_TESTS: + mock_response_from_jsonl(name, 'test_cli') + return f(*args, **kwargs) + + return conditional_mock_responses(auto_mock_data=False)(wrapper) + + return decorator + + +def run(capsys, cmdline): + main(args=cmdline.split()) + out, err = capsys.readouterr() + assert DEPRECATED in err and 'suggested replacement: cdxt ' in err + return out, err + + +@recorded('test_basics') +def test_basics(capsys): + out, err = run(capsys, '--cc --limit 10 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 10 + for line in lines: + assert TEXT_LINE.fullmatch(line) and 'commoncrawl.org' in line + assert ("cdxt index cc --url 'commoncrawl.org/*' --limit 10 --output-format csv --fields status,timestamp,url" + in err) + + +@recorded('test_basics_2') +def test_basics_2(capsys): + out, _ = run(capsys, '--crawl 2 --limit 10 iter commoncrawl.org/*') + assert len(out.splitlines()) == 10 + + +@recorded('test_multi_cc1_slow_4') +def test_all_fields(capsys): + out, _ = run(capsys, '--cc --limit 10 iter commoncrawl.org/* --all-fields') + lines = out.splitlines() + assert len(lines) == 10 and all('digest ' in line and 'filename crawl-data/' in line for line in lines) + + +@recorded('test_multi_cc1_slow_5') +def test_csv(capsys): + out, _ = run(capsys, '--cc --limit 10 iter commoncrawl.org/* --fields=digest,length,offset,digest --csv') + lines = out.split('\r\n') + assert lines[0] == 'digest,length,offset' + assert len(lines) == 12 and lines[-1] == '' # header, 10 captures, CRLF line endings + assert all(line.count(',') == 2 for line in lines[:-1]) + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_cc2_slow_4') +def test_jsonl(capsys): + out, _ = run(capsys, '--cc --limit 10 iter commoncrawl.org/* --jsonl') + lines = out.splitlines() + assert len(lines) == 10 + assert all(list(json.loads(line)) == ['status', 'timestamp', 'url'] for line in lines) + + +@recorded('test_multi_cc1_slow_7') +def test_filters(capsys): + out, _ = run(capsys, '--cc --limit 10 --filter=!status:200 --filter=!status:404 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 10 and not any(line.startswith(('status 200,', 'status 404,')) for line in lines) + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_cc1_slow_9') +def test_from_to(capsys): + out, _ = run(capsys, '--cc --limit 10 --from=2017 --to=2017 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 10 and all(', timestamp 2017' in line for line in lines) + + +@recorded('test_multi_cc2') +def test_get_closest(capsys): + out, _ = run(capsys, '--cc --limit 3 --get --closest=20170615 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 3 and all(', timestamp 20170' in line for line in lines) + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_cc2_slow_2') +def test_get_without_limit(capsys): + # the recorded requests have limit=1000, the default of a single get + out, err = run(capsys, '--cc --get --closest=20170615 iter commoncrawl.org/never-existed') + assert out == '' + assert '--limit 1000 --closest 20170615' in err + + +@recorded('test_multi_ia') +def test_ia(capsys): + out, _ = run(capsys, '--ia --limit 10 iter commoncrawl.org/*') + assert len(out.splitlines()) == 10 + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_misc_ia') +def test_source(capsys): + out, _ = run(capsys, '--source https://web.archive.org/cdx/search/cdx --limit 10 iter commoncrawl.org/*') + assert len(out.splitlines()) == 10 + + +@recorded('test_multi_misc_not_ia_slow_4') +def test_size(capsys): + out, err = run(capsys, '--cc --limit 10 size commoncrawl.org/*') + assert out.strip().isdigit() + assert "cdxt index cc --url 'commoncrawl.org/*' --limit 10 --count" in err + + +@responses.activate +def test_size_details(capsys): + # one CDX API server: the estimate of the index, then the total (not capped by --limit) + responses.add(responses.GET, 'https://example.com/cdx', json={'blocks': 3}) + out, _ = run(capsys, '--source https://example.com/cdx --limit 10 size --details commoncrawl.org/*') + assert out == 'https://example.com/cdx 6000\n6000\n' + assert responses.calls[0].request.params == {'url': 'commoncrawl.org/*', 'showNumPages': 'true', 'limit': '10'} + + +def warc_records(path): + """(type, content) of the records of a WARC file.""" + with open(path, 'rb') as f: + return [(record.rec_type, record.content_stream().read().decode(errors='replace')) + for record in ArchiveIterator(f)] + + +@recorded('test_warc_prefix_1') +def test_warc_cc(tmpdir, capsys): + with tmpdir.as_cwd(): + _, err = run(capsys, '-v -v --cc --limit 2 warc commoncrawl.org/*') + assert [p.basename for p in tmpdir.listdir()] == ['TEST-001.warc.gz'] + records = warc_records(tmpdir / 'TEST-001.warc.gz') + assert [rec_type for rec_type, _ in records] == ['warcinfo', 'response', 'response'] + warcinfo = records[0][1] + assert 'isPartOf: TEST' in warcinfo + assert 'description: warc extraction generated with: cdxt -v -v --cc --limit 2 warc commoncrawl.org/*' in warcinfo + assert "cdxt index cc --url 'commoncrawl.org/*' --limit 2 | cdxt repackage --prefix TEST --no-sort" in err + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_warc_prefix_2') +def test_warc_ia(tmpdir, capsys): + # replayed from the Internet Archive's wayback machine, without --wb + with tmpdir.as_cwd(): + _, err = run(capsys, '--ia --limit 1 warc commoncrawl.org/*') + assert [p.basename for p in tmpdir.listdir()] == ['TEST-001.warc.gz'] + assert [rec_type for rec_type, _ in warc_records(tmpdir / 'TEST-001.warc.gz')] == ['warcinfo', 'response'] + assert '| cdxt repackage --prefix TEST --wb ia --no-sort' in err + + +@recorded('test_warc_suffix_1') +def test_warc_options(tmpdir, capsys): + with tmpdir.as_cwd(): + run(capsys, '-v -v --cc --limit 1 warc --prefix FOO --subprefix BAR --size 1 --creator creator ' + '--operator bob --url-fgrep common --url-fgrepv bar commoncrawl.org/*') + assert [p.basename for p in tmpdir.listdir()] == ['FOO-BAR-001.warc.gz'] + records = warc_records(tmpdir / 'FOO-BAR-001.warc.gz') + warcinfo = records[0][1] + assert 'isPartOf: FOO-BAR' in warcinfo and 'creator: creator' in warcinfo and 'operator: bob' in warcinfo + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_warc_suffix_2') +def test_warc_empty_selection(tmpdir, capsys): + with tmpdir.as_cwd(): + run(capsys, '-v -v --cc --limit 1 warc --prefix EMPTY --size 1 --url-fgrep bar commoncrawl.org/*') + assert tmpdir.listdir() == [] # no WARC file without captures diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index c34832f..7be322f 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -11,7 +11,6 @@ import cdx_toolkit.warc from cdx_toolkit.cli import main from cdx_toolkit.cli.common import parse_size -from cdx_toolkit.cli.deprecated import get_command, rewrite_deprecated_command, translate from cdx_toolkit.repackage.jobs import RepackageResult import cdx_toolkit.cli.index @@ -255,51 +254,6 @@ def test_parse_size_invalid(value): parse_size(value) -@pytest.mark.parametrize( - 'args, command', - [ - ('--cc --limit 10 iter example.com', 'iter'), - ('--source https://example.com/cdx --crawl 3 size example.com', 'size'), - ('-v index cc --url example.com', 'index'), - ('--version', None), - ], -) -def test_get_command(args, command): - assert get_command(args.split()) == command - - -@pytest.mark.parametrize( - 'args, expected', - [ - ('iter cc --url example.com/*', ['index', 'cc', '--url', 'example.com/*']), - ('-v iter --help', ['-v', 'index', '--help']), - # the old syntax is not translated, the arguments are the same as for `index` - ('--cc --limit 10 iter example.com/*', ['--cc', '--limit', '10', 'index', 'example.com/*']), - ], -) -def test_iter_is_an_alias_of_index(capsys, args, expected): - assert rewrite_deprecated_command(args.split()) == expected - assert '"iter" is deprecated' in capsys.readouterr().err - - -def test_iter_runs_like_index(capsys): - with patch.object(cdx_toolkit.cli.index, 'run') as run: - main(['iter', 'cc', '--url', 'example.com/*', '--limit', '3', '-o', 'out.csv']) - - args = run.call_args.args[0] - assert (args.cmd, args.index, args.urls) == ('index', 'cc', ['example.com/*']) - assert (args.limit, args.output) == (3, 'out.csv') - assert '"iter" is deprecated' in capsys.readouterr().err - - -def test_translate_warc(): - args = '--source https://example.com/cdx --wb https://example.com/wb --limit=5 warc --size 1000 example.com/*' - assert translate(args.split()) == ( - "cdxt index https://example.com/cdx --url 'example.com/*' --limit 5" - " | cdxt repackage --wb https://example.com/wb --size 1000" - ) - - def test_verbose_before_and_after_subcommand(tmp_path): captures = _write_captures(tmp_path / 'captures.jsonl', []) with patch.object(cdx_toolkit.cli, 'set_loglevel') as set_loglevel: diff --git a/tests/unit/test_cli_legacy.py b/tests/unit/test_cli_legacy.py new file mode 100644 index 0000000..fb2397f --- /dev/null +++ b/tests/unit/test_cli_legacy.py @@ -0,0 +1,286 @@ +'''The command lines of cdx_toolkit 0.9.39 (cdx_toolkit/cli/legacy.py), without requests.''' +import json +import shlex +from unittest.mock import Mock, patch + +import pytest + +import cdx_toolkit.cli.legacy as legacy +from cdx_toolkit.cli import build_parser, main +from cdx_toolkit.cli.legacy import build_legacy_parser, is_legacy_command_line, suggested_replacement +from cdx_toolkit.repackage.jobs import RepackageResult + + +CAPTURES = [ + {'url': 'https://example.com/', 'status': '200', 'timestamp': '20240101000000', 'digest': 'A', 'mime': 'text/html'}, + {'url': 'https://example.com/b', 'status': '404', 'timestamp': '20240102000000', 'digest': 'B'}, +] + + +def parse(cmdline: str): + return build_legacy_parser().parse_args(shlex.split(cmdline)) + + +@pytest.mark.parametrize( + 'args, expected', + [ + ('--cc --limit 10 iter example.com/*', True), + ('--crawl 2 iter example.com', True), + ('--limit=5 iter example.com', True), + ('-v -v --cc size example.com', True), + ('--ia warc example.com', True), + ('iter cc --url example.com', True), # the unreleased alias: rejected by the old grammar + ('--limit iter example.com', True), # old options before the command: strict old grammar (error) + ('-- iter example.com', True), + ('index cc --url example.com', False), + ('-v index cc --url iter', False), + ('-vv repackage captures.jsonl', False), + ('index cc --crawl iter --url example.com', False), + ('--version', False), + ('-h', False), + ('', False), + ], +) +def test_is_legacy_command_line(args, expected): + assert is_legacy_command_line(args.split()) == expected + + +def test_a_value_of_an_old_option_is_not_the_command(): + # `--source iter`: iter is the value of --source, the command is size + assert is_legacy_command_line(['--source', 'iter', 'size', 'x']) + ns = parse('--source https://example.com/cdx --filter iter size x') + assert (ns.cmd, ns.filter) == ('size', ['iter']) + + +@pytest.mark.parametrize( + 'args, message', + [ + ('--cc iter', 'the following arguments are required: url'), + ('--cc iter --url example.com', 'unrecognized arguments: --url'), + ('iter cc --url example.com', 'unrecognized arguments: --url'), + ('--limit iter example.com', "invalid int value: 'iter'"), + ('--cc --limit', 'expected one argument'), + ('--cc iter --size 3 example.com', 'unrecognized arguments: --size'), + ('--limit 3 iter example.com', 'must specify --cc, --ia, or a --source'), + ('--source example.com iter example.com', 'use the http(s) URL of a CDX API server'), + ('--cc iter --all-fields --csv example.com', '--all-fields cannot be used with --csv'), + ('--cc --wb https://example.com/wb warc example.com', 'cannot specify --wb for Common Crawl'), + ('--source https://example.com/cdx warc example.com', 'needs --wb (a wayback machine) or'), + ('--ia --wb ia warc --warc-download-prefix https://example.com example.com', 'cannot specify both'), + ], +) +def test_invalid_old_command_lines_exit_2_without_requests(capsys, args, message): + with patch.object(legacy, 'open_index') as open_index, patch.object(legacy, 'fetcher') as fetcher, \ + patch('cdx_toolkit.repackage.repackage') as repackage: + with pytest.raises(SystemExit) as e: + main(args.split()) + assert e.value.code == 2 + assert message in capsys.readouterr().err + open_index.assert_not_called() + fetcher.assert_not_called() + repackage.assert_not_called() + + +@pytest.mark.parametrize( + 'args, source', + [ + ('--crawl 1 --ia iter x', 'cc'), + ('--cc --ia iter x', 'cc'), + ('--source https://example.com/cdx --ia iter x', 'ia'), + ('--source https://example.com/cdx iter x', 'https://example.com/cdx'), + ], +) +def test_source_precedence(args, source): + assert legacy.legacy_source(parse(args)) == source + + +def run_iter(capsys, args, captures=CAPTURES): + """Run an old iter command line on `captures` from the CDX API backend.""" + backend = Mock() + backend.records.return_value = iter(captures) + with patch.object(legacy, 'open_index', return_value=backend) as open_index: + main(args.split()) + out, err = capsys.readouterr() + return out, err, open_index, backend + + +def test_iter_text_output(capsys): + out, err, open_index, backend = run_iter( + capsys, '--cc --crawl CC-MAIN-2024-33 --limit 2 --from 2024 --filter =status:200 iter example.com/*' + ) + assert out.splitlines() == [ + 'status 200, timestamp 20240101000000, url https://example.com/', + 'status 404, timestamp 20240102000000, url https://example.com/b', + ] + assert open_index.call_args.args == ('cc', 'cdx-api') + query = backend.records.call_args.args[0] + assert (query.urls, query.crawl, query.limit, query.from_ts, query.filters) == ( + ['example.com/*'], 'CC-MAIN-2024-33', 2, '2024', ['=status:200']) + assert err.count('\n') == 1 + assert 'the old syntax, which is deprecated and will be removed in a future version' in err + assert ("suggested replacement: cdxt index cc --url 'example.com/*' --crawl CC-MAIN-2024-33 --limit 2 " + '--from 2024 --filter =status:200 --output-format csv --fields status,timestamp,url') in err + + +def test_iter_fields_are_deduplicated_and_missing_ones_omitted(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --fields mime,url,mime x') + assert out.splitlines() == ['mime text/html, url https://example.com/', 'url https://example.com/b'] + + +def test_iter_all_fields(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --fields url --all-fields x') + assert out.splitlines()[0] == ( + 'digest A, mime text/html, status 200, timestamp 20240101000000, url https://example.com/' + ) + + +def test_iter_jsonl(capsys): + out, err, _, _ = run_iter(capsys, '--ia iter --jsonl x') + assert out.splitlines()[0] == '{"status": "200", "timestamp": "20240101000000", "url": "https://example.com/"}' + assert "cdxt index ia --url x --fields status,timestamp,url (see" in err + + +def test_iter_jsonl_all_fields(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --jsonl --all-fields x') + assert json.loads(out.splitlines()[1]) == CAPTURES[1] + + +def test_iter_csv(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --csv --fields url,digest,mime x') + assert out == 'digest,mime,url\r\nA,text/html,https://example.com/\r\nB,,https://example.com/b\r\n' + + +def test_iter_csv_header_without_captures(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --csv x', captures=[]) + assert out == 'status,timestamp,url\r\n' + + +def test_iter_jsonl_and_csv(capsys): + # as in 0.9.39: the CSV header, then JSON lines + out, _, _, _ = run_iter(capsys, '--ia iter --csv --jsonl x') + assert out.splitlines()[:2] == [ + 'status,timestamp,url', '{"status": "200", "timestamp": "20240101000000", "url": "https://example.com/"}' + ] + + +@pytest.mark.parametrize( + 'args, method, kwargs', + [ + ('--cc --get iter x', 'get', {}), # CDXFetcher.get() adds limit=1000 + ('--cc --get --limit 3 --closest 2017 iter x', 'get', {'limit': 3, 'closest': '2017'}), + ('--cc --closest 2017 --to 2018 iter x', 'iter', {'closest': '2017', 'to': '2018'}), # paged, as before + ], +) +def test_iter_get_and_closest_use_the_fetcher(capsys, args, method, kwargs): + cdx = Mock() + getattr(cdx, method).return_value = CAPTURES[:1] + with patch.object(legacy, 'fetcher', return_value=cdx), patch.object(legacy, 'open_index') as open_index: + main(args.split()) + getattr(cdx, method).assert_called_once_with('x', **kwargs) + open_index.assert_not_called() + out, err = capsys.readouterr() + assert out == 'status 200, timestamp 20240101000000, url https://example.com/\n' + if args == '--cc --get iter x': + assert 'cdxt index cc --url x --limit 1000 --output-format csv' in err + + +def test_fetcher_options(): + with patch('cdx_toolkit.CDXFetcher') as fetcher: + legacy.fetcher(parse('--crawl 2 --cc-mirror https://m/ --cc-sort ascending iter x'), 'cc') + assert fetcher.call_args.kwargs == { + 'source': 'cc', 'crawl': ['2'], 'cc_mirror': 'https://m/', 'cc_sort': 'ascending'} + legacy.fetcher(parse('--ia --cc-mirror https://m/ iter x'), 'ia') # ignored for other sources, as before + assert fetcher.call_args.kwargs == {'source': 'ia'} + + +def test_size_is_not_capped_by_the_limit(capsys): + cdx = Mock() + cdx.get_size_estimate.return_value = 1500 + with patch.object(legacy, 'fetcher', return_value=cdx): + main('--cc --limit 10 size --details x'.split()) + cdx.get_size_estimate.assert_called_once_with('x', limit=10, details=True) + out, err = capsys.readouterr() + assert out == '1500\n' + assert "suggested replacement: cdxt index cc --url x --limit 10 --count" in err + + +def run_warc(args, result=RepackageResult(2, 2, 0), captures=CAPTURES): + with patch.object(legacy, 'legacy_captures', return_value=iter(captures)) as legacy_captures, \ + patch('cdx_toolkit.repackage.repackage', return_value=result) as repackage: + main(args.split()) + return legacy_captures, repackage + + +def test_warc_streams_into_repackage(capsys): + legacy_captures, repackage = run_warc( + '--cc --get --limit 2 warc --subprefix SUB --size 1000 --creator me --url-fgrepv /b x') + assert legacy_captures.call_args.kwargs == {'use_get': False} # 0.9.39 iterated with --get too + args, kwargs = repackage.call_args + assert list(args[0]) == CAPTURES[:1] # --url-fgrepv after the limit + assert args[1] == 'TEST' + assert kwargs['sort'] is False + assert kwargs['warc_download_prefix'] == 'https://data.commoncrawl.org' and 'wb' not in kwargs + assert (kwargs['subprefix'], kwargs['size'], kwargs['creator'], kwargs['operator']) == ('SUB', 1000, 'me', None) + assert kwargs['description'] == ( + 'warc extraction generated with: cdxt --cc --get --limit 2 warc --subprefix SUB --size 1000 --creator me ' + '--url-fgrepv /b x' + ) + assert ('suggested replacement: cdxt index cc --url x --limit 2 --url-fgrepv /b | cdxt repackage --prefix TEST ' + '--subprefix SUB --size 1000 --creator me --no-sort') in capsys.readouterr().err + + +@pytest.mark.parametrize( + 'args, content, suggestion', + [ + ('--ia warc x', {'wb': 'ia'}, '| cdxt repackage --prefix TEST --wb ia --no-sort'), + ('--ia warc --warc-download-prefix https://w x', {'warc_download_prefix': 'https://w'}, + '--warc-download-prefix https://w --no-sort'), + ('--source https://e/cdx --wb https://e/wb warc x', {'wb': 'https://e/wb'}, '--wb https://e/wb --no-sort'), + ('--crawl 1 warc --warc-download-prefix s3://commoncrawl x', {'warc_download_prefix': 's3://commoncrawl'}, + '--warc-download-prefix s3://commoncrawl --no-sort'), + ], +) +def test_warc_content_source(capsys, args, content, suggestion): + _, repackage = run_warc(args) + kwargs = repackage.call_args.kwargs + assert {k: kwargs[k] for k in ('wb', 'warc_download_prefix') if k in kwargs} == content + assert suggestion in capsys.readouterr().err + + +def test_warc_exits_1_if_captures_failed(capsys): + with pytest.raises(SystemExit) as e: + run_warc('--cc warc x', result=RepackageResult(2, 1, 1)) + assert e.value.code == 1 + assert '1 of 2 captures could not be fetched' in capsys.readouterr().err + + +@pytest.mark.parametrize( + 'args', + [ + '--cc --limit 10 iter example.com/*', + "--crawl 'CC-MAIN-2024-33,CC-MAIN-2024-30' --cc-sort ascending --from 2024 --to 20240410 " + "--filter '!=status:200' --filter =mime:text/html iter --fields url,digest example.com/*", + '--ia --get --closest 2017 iter --jsonl --all-fields example.com', + '--source https://example.com/cdx iter --csv example.com', + '--cc --limit 3 size --details example.com', + "--ia warc --prefix 'my dir/IA' --subprefix X --size 100 --creator 'a b' --operator c --url-fgrep a " + '--url-fgrepv robots.txt example.com/*', + '--cc --cc-mirror https://index.commoncrawl.org/ warc --warc-download-prefix s3://commoncrawl example.com', + ], +) +def test_suggested_replacement_is_a_valid_new_command_line(args): + suggestion = suggested_replacement(parse(args)) + for part in suggestion.split(' | '): + tokens = shlex.split(part) + assert tokens[0] == 'cdxt' + build_parser().parse_args(tokens[1:]) # exits on an error + + +def test_legacy_captures_without_a_crawl_or_limit(): + backend = Mock() + with patch.object(legacy, 'open_index', return_value=backend) as open_index: + legacy.legacy_captures(parse('--source https://example.com/cdx.php iter x'), 'https://example.com/cdx.php', + use_get=False) + assert open_index.call_args.args == ('https://example.com/cdx.php', 'cdx-api') # not detected from the URL + query = backend.records.call_args.args[0] + assert (query.crawl, query.limit, query.filters) == (None, None, []) From 721fd4c714dbbdd8f359aa0bd892c4e90a0437e6 Mon Sep 17 00:00:00 2001 From: malteos Date: Fri, 16 Oct 2026 12:01:56 +0200 Subject: [PATCH 89/91] docs: get_writer() will be removed in a future version --- cdx_toolkit/warc.py | 2 +- docs/cli-migration-guide.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdx_toolkit/warc.py b/cdx_toolkit/warc.py index 06a9026..4a99bc9 100644 --- a/cdx_toolkit/warc.py +++ b/cdx_toolkit/warc.py @@ -208,7 +208,7 @@ class CDXToolkitWARCWriter: def __init__(self, prefix, subprefix, info, size=1000000000, gzip=True, warc_version=None): warnings.warn( 'cdx_toolkit.warc.get_writer() and CDXToolkitWARCWriter are deprecated and will be removed in ' - 'cdx_toolkit 1.0; use cdx_toolkit.repackage.repackage()', + 'a future version; use cdx_toolkit.repackage.repackage()', FutureWarning, stacklevel=2, ) diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index 1101045..b6893ac 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -320,4 +320,4 @@ repackage(index.records(query), prefix='COMMONCRAWL') # | cdxt repackage ``` `cdx_toolkit.warc.get_writer()` (`CDXToolkitWARCWriter`) is deprecated and will be removed in -cdx_toolkit 1.0; `repackage()` replaces it. +a future version; `repackage()` replaces it. From b27e7c3bdc8e1eca2481500374c26cf908ca5c46 Mon Sep 17 00:00:00 2001 From: malteos Date: Fri, 16 Oct 2026 12:13:43 +0200 Subject: [PATCH 90/91] docs: --crawl with SQL indexes, Parquet output, TEST prefix of the old warc in the migration guide --- docs/cli-migration-guide.md | 15 ++++++++------- docs/settings.md | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index b6893ac..0ba4de3 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -2,7 +2,7 @@ The `cdxt` command line tool has a new design with two commands: -- **`cdxt index`** selects captures from an index and writes them as JSONL, CDXJ, CSV or TSV +- **`cdxt index`** selects captures from an index and writes them as JSONL, CDXJ, CSV, TSV or Parquet (replaces `cdxt iter` and `cdxt size`). - **`cdxt repackage`** fetches the captures listed in an index file and writes them into WARC files (replaces `cdxt warc`, together with `cdxt index`). @@ -45,8 +45,8 @@ The old `warc` command now runs like `cdxt index ... | cdxt repackage --no-sort` | `cdxt --source CDX_API_URL iter URL` | `cdxt index CDX_API_URL --url URL` | | `cdxt --crawl 3 iter URL` | `cdxt index cc --crawl 3 --url URL` | | `cdxt --cc size URL` | `cdxt index cc --url URL --count` | -| `cdxt --cc warc URL` | `cdxt index cc --url URL \| cdxt repackage` | -| `cdxt --ia warc URL` | `cdxt index ia --url URL \| cdxt repackage --wb ia` | +| `cdxt --cc warc URL` | `cdxt index cc --url URL \| cdxt repackage --prefix TEST` | +| `cdxt --ia warc URL` | `cdxt index ia --url URL \| cdxt repackage --prefix TEST --wb ia` | ## What changed @@ -75,7 +75,8 @@ cdxt index cc --url 'commoncrawl.org/*' --limit 10 --from 2017 --output-format c | `--source https://web.archive.org/cdx/search/cdx` | `cdxt index https://web.archive.org/cdx/search/cdx` | | `--crawl ...` (implied `--cc`) | `cdxt index cc --crawl ...` | -`--crawl` now has to be combined with `cc` explicitly; it cannot be used with other indexes. +`--crawl` now has to be combined with `cc` explicitly (or a columnar index such as `cc-url-index-s3`); it +cannot be used with `ia` or other CDX API servers. ### The URL is an option @@ -102,12 +103,12 @@ cdxt index cc --domain commoncrawl.org --url 'example.com/about*' --limit 100 | `iter` (text: `status 200, timestamp ..., url ...`) | JSONL by default | | `iter --jsonl` | default, or `--output-format jsonl` | | `iter --csv` | `--output-format csv`, or `-o captures.csv` | -| — | `--output-format cdxj` / `tsv`, or `-o captures.cdxj` / `captures.tsv` | +| — | `--output-format cdxj` / `tsv` / `parquet`, or `-o captures.cdxj` / `captures.tsv` / `captures.parquet` | | `--fields url,status,timestamp` (default) | `--fields url,status,timestamp` (default: all fields) | | `--all-fields` | default | The output format is detected from the `--output` file extension (`.jsonl`, `.json`, `.cdxj`, -`.cdx`, `.csv`, `.tsv`, each optionally with `.gz`). For other extensions, set `--output-format`. +`.cdx`, `.csv`, `.tsv`, each optionally with `.gz`, or `.parquet`, which needs `cdx_toolkit[parquet]`). For other extensions, set `--output-format`. The old human-readable text output is gone. The closest replacement is CSV with the old default fields: @@ -171,7 +172,7 @@ The `warc` options are split between the two commands: **Common Crawl** captures contain their WARC location (`filename`, `offset`, `length`) and are fetched from `--warc-download-prefix` (default `https://data.commoncrawl.org`; use -`s3://commoncrawl` in AWS us-east-1). Two things differ from the old `warc` command: +`s3://commoncrawl` in AWS us-east-1). Three things differ from the old `warc` command: - **File names:** `[-]-001.warc.gz`, `-002`, ... instead of `[-]-000000.extracted.warc.gz`, `-000001`, ... diff --git a/docs/settings.md b/docs/settings.md index 0bacc90..ab3b4e4 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -11,7 +11,7 @@ Requests over HTTP(S) go through `cdx_toolkit/myrequests.py`: default `--warc-download-prefix`, and `capture.content` in Python); - wayback replays, e.g. from `https://web.archive.org/web` (`cdxt repackage --wb ia`). -Reads from `s3://` (see above), `hf://`, and the queries of DuckDB and Athena do not use it. +Reads from `s3://` (see [S3 settings](#s3-settings)), `hf://`, and the queries of DuckDB and Athena do not use it. ### Minimum interval between requests From 92c76e6e53af98e31ee9beb998cde94a6112d805 Mon Sep 17 00:00:00 2001 From: malteos Date: Fri, 16 Oct 2026 14:32:46 +0200 Subject: [PATCH 91/91] docs: repository URLs of commoncrawl/cdx_toolkit; CHANGELOG entry of 0.9.39 --- CHANGELOG.md | 9 +++++++++ README.md | 2 +- setup.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13e97a2..08a8ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,15 @@ + new extras: `athena`, `duckdb`, `hf` (`huggingface_hub>=1.7.2`, the first version whose `HfFileSystem` handles Storage Buckets), `parquet`; `fsspec` and `aiofiles` are now required +- 0.9.39 + + `CDXToolkitWARCWriter` (`cdx_toolkit.warc.get_writer()`) supports the context manager protocol (`with ... as writer:`), + so the WARC file is closed, and an S3 upload committed, also on errors; `close()` still works (#70) + + WARC files can be written to S3 (and other fsspec file systems); `cdx_toolkit[s3]` extra (#67) + + settings are read from `CDXT_*` environment variables (`settings.py`), common CLI code moved to `utils.py`, + the CLI and the example close the WARC writer (#67) + + weekly GitHub Action that checks Common Crawl's CDX index server (#60) + + PyPI releases with a manually dispatched trusted-publisher workflow (#69) + - 0.9.38 + deprecated support for py3.7 and py.3.8 + added support for py3.13 and py3.14 diff --git a/README.md b/README.md index ea88ee0..f764212 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # cdx_toolkit -[![build](https://github.com/cocrawler/cdx_toolkit/actions/workflows/ci.yaml/badge.svg)](https://github.com/cocrawler/cdx_toolkit/actions/workflows/ci.yaml) [![coverage](https://codecov.io/gh/cocrawler/cdx_toolkit/graph/badge.svg?token=M1YJB998LE)](https://codecov.io/gh/cocrawler/cdx_toolkit) [![Apache License 2.0](https://img.shields.io/github/license/cocrawler/cdx_toolkit.svg)](LICENSE) +[![build](https://github.com/commoncrawl/cdx_toolkit/actions/workflows/ci.yaml/badge.svg)](https://github.com/commoncrawl/cdx_toolkit/actions/workflows/ci.yaml) [![coverage](https://codecov.io/gh/commoncrawl/cdx_toolkit/graph/badge.svg?token=M1YJB998LE)](https://codecov.io/gh/commoncrawl/cdx_toolkit) [![Apache License 2.0](https://img.shields.io/github/license/commoncrawl/cdx_toolkit.svg)](LICENSE) cdx_toolkit is a set of tools for working with CDX indices of web crawls and archives, including those at the Common Crawl Foundation diff --git a/setup.py b/setup.py index 22fc5ec..9dfec6f 100755 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ long_description_content_type='text/markdown', author='Greg Lindahl and others', author_email='lindahl@pbm.com', - url='https://github.com/cocrawler/cdx_toolkit', + url='https://github.com/commoncrawl/cdx_toolkit', packages=packages, python_requires=">=3.9", extras_require=extras_require,