diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7b9efa3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# test fixtures are compared byte for byte (e.g. WARC payload digests): no line-ending conversion +tests/data/** -text diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml index ae7f3c5..5c3f433 100644 --- a/.github/workflows/ci-slow.yaml +++ b/.github/workflows/ci-slow.yaml @@ -2,6 +2,14 @@ name: CI-slow on: workflow_dispatch +# live tests (CDXT_LIVE_TESTS=http,s3,athena,hf): the HTTP requests to the CDX APIs and data.commoncrawl.org, S3, +# Athena and DuckDB against AWS, and the Hugging Face Storage Buckets (HF_TOKEN: writes to +# hf://buckets/commoncrawl/commoncrawl-ci-temp). The permissions are needed to get AWS credentials via GitHub's OIDC +# token endpoint +permissions: + id-token: write + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.ref }} # 'queue: max' ensures old runs finish in order instead of being canceled @@ -42,8 +50,17 @@ jobs: pip install setuptools - name: Install cdx_toolkit - run: pip install .[test] + run: pip install ".[all]" + + - name: Configure AWS credentials from OIDC + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::837454214164:role/GitHubActions-Role + aws-region: us-east-1 - name: Run tests + env: + CDXT_LIVE_TESTS: 'http,s3,athena,hf' + HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | make test_coverage diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3755861..a9a524f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,9 +8,8 @@ on: branches: - main -# These permissions are needed to interact with AWS S3 via GitHub's OIDC Token endpoint +# the tests make no remote requests (they are mocked); the live tests run in ci-slow.yaml permissions: - id-token: write contents: read pull-requests: read @@ -39,7 +38,7 @@ jobs: - python-version: '3.14' os: ubuntu-latest env: - LOGLEVEL=DEBUG + CDXT_LOGLEVEL=DEBUG EXTRA: true - python-version: '3.14' os: macos-latest @@ -68,21 +67,8 @@ jobs: pip install setuptools - name: Install cdx_toolkit - run: pip install .[all] + run: pip install ".[all]" - - name: Configure AWS credentials from OIDC (disabled for forks) - if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push' - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: arn:aws:iam::837454214164:role/GitHubActions-Role - aws-region: us-east-1 - - - name: Disable S3 unit tests for Python 3.8 (botocore requires Python 3.9+) - if: ${{ startsWith(matrix.python-version, '3.8') }} - uses: actions/github-script@v9 - with: - script: | - core.exportVariable('CDXT_DISABLE_S3_TESTS', '1') - name: Set environment variables for faster unit tests (requests are mocked) uses: actions/github-script@v9 with: @@ -93,8 +79,7 @@ jobs: core.exportVariable('CDXT_CC_INDEX_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01') - core.exportVariable('DISABLE_ATHENA_TESTS', '1') - core.exportVariable('LOGLEVEL', 'DEBUG') + core.exportVariable('CDXT_LOGLEVEL', 'DEBUG') - name: Lint code run: | @@ -140,7 +125,7 @@ jobs: pip install setuptools - name: Install cdx_toolkit (minimal) - run: pip install .[test] + run: pip install ".[test]" - name: Set environment variables for faster unit tests (requests are mocked) uses: actions/github-script@v9 @@ -152,8 +137,7 @@ jobs: core.exportVariable('CDXT_CC_INDEX_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01') core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01') - core.exportVariable('DISABLE_ATHENA_TESTS', '1') - core.exportVariable('LOGLEVEL', 'DEBUG') + core.exportVariable('CDXT_LOGLEVEL', 'DEBUG') - name: test minimal run: | diff --git a/.travis.yml b/.travis.yml index 6401533..f52c97e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: arch: s390x # unreliable in travis include: - python: 3.5 - env: LOGLEVEL=DEBUG # same time as normal + env: CDXT_LOGLEVEL=DEBUG # same time as normal - python: 3.8 arch: arm64 # 50% extra runtime - python: 3.8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cfe21c..08a8ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,64 @@ +- unreleased + + new command line interface: `cdxt index` and `cdxt repackage` (see README) + + the command lines of 0.9.39 (`cdxt --cc iter URL`, `size`, `warc`) are deprecated (to be removed in a future version): they run + with the new code and print a suggested new command line; `iter` output, `--get` and `size --details` are kept; + `warc` writes `TEST-001.warc.gz` (was `TEST-000000.extracted.warc.gz`) with the records copied byte-for-byte + (no `WARC-Source-URI`/`WARC-Source-Range` headers), no file without captures, and exits with status 1 if + captures could not be fetched; `iter --all-fields --csv` is a usage error (was a traceback) + + `cdxt index`: `--hostname`, `--domain`, repeatable `--url`, `-o/--output` (local or remote), + JSONL/CDXJ/CSV/TSV output, all fields by default, `--count` replaces `size` + + `cdxt repackage`: reads captures from a file or stdin, human-readable `--size`, `--is-part-of`, `--description` + + `--cc-sort` is now passed on to the CDX fetcher + + the log level variable is `CDXT_LOGLEVEL` (`LOGLEVEL` still works), like cdx_toolkit's other environment variables + + `cdxt index` queries Common Crawl's columnar URL index with DuckDB (`cc-url-index-s3`, `cc-url-index-hf`, `*.parquet`) + or AWS Athena (`athena://...`): `--hostname`/`--domain`/`--crawl`/`--limit` or your own `--sql`, cost confirmation (`--yes`), `--dry-run`; + `--index-backend cdx-api|cdx-files|duckdb|athena` overrides the backend detected from INDEX + + `cdxt repackage` copies WARC records byte-for-byte with the async WARC fetcher (from the feat/warc-range-sources branch): + reads from https://, s3:// and hf:// (Hugging Face Storage Buckets), writes to local, s3:// or hf://, + `--processes`, `--parallel-readers`, `--add-metadata-record`, glob input; output files are now named `-001.warc.gz` + (`--wb` wayback replays too, which now work with `--parallel-readers` and `--add-metadata-record`); with + `--processes`, the processes share one file counter + and write the final file names directly, so each `warcinfo` record names its own file + + `hf://` WARC reads resolve each WARC file once and read its records from the signed CDN URL + (one HF Resolvers rate-limit request per WARC file instead of per record) + + `cdxt index` reads CDX files (local or remote, globs): `--url`/`--hostname`/`--domain`/`--url-file` matched as SURT + prefixes (`--url-file-type domain` for host lists), one output per CDX file for an `-o` path without extension + (a new or empty directory), `--processes`; `cdxt index` refuses an output that would overwrite its input + + SQL indexes: `--url` patterns (matched against `url_surtkey`), `--from`/`--to` (fetch time), `--language` (primary content language, three-letter ISO 639-3 codes) + + SQL indexes: `--hostname`, `--domain` and `--url` select by `url_host_name_reversed`, which follows the sort order + of the index, so DuckDB (and Athena) skip the row groups of other hosts by their min/max statistics; `--domain` no + longer reads `url_surtkey` (one Parquet file: 3 s instead of 87 s over S3) + + `--output-format cdx` / `--input-format cdx` as another name for `cdxj`; clear error for globs over http(s) + + Parquet output for `cdxt index` and Parquet input for `cdxt repackage` (`cdx_toolkit[parquet]`) + + `cdxt repackage`: default `--prefix REPACKAGE` (was `TEST`), no WARC file is written without captures, + a full URL in `warc_filename`/`filename` is used as it is (the `warc_url` column is gone); stops (exit status 2) + if WARC files with the same prefix exist already, instead of overwriting them (also with `--wb`, which + numbered its files after the existing ones before) + + fixed `--to` with a day (e.g. `--to 20240410`), which was padded to the end of the month, and February in leap years + + `cdxt index ia --filter` with a filter the Internet Archive does not support (`=`, `~`, `!=`, `!~`) is a usage error + (exit status 2) before any request, instead of a traceback while the captures are read + + fixed DNS errors of hosts that worked before: they are retried again (like connection errors) instead of failing at once + + removed the deprecated `cdx_iter` and `cdx_size` scripts + + S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual + addressing, no payload signing, 2 concurrent requests) unless configured by the user; `CDXT_AWS_DEFAULTS=0` turns them off; + also for Athena and, as far as DuckDB has equivalents, for DuckDB (fixes DuckDB's `http_timeout`, which is in seconds) + + `cdxt repackage --wb` checks the captures before replaying them: `url` (http/https), `timestamp` and `status` + are needed, and Common Crawl captures are refused (as `cdxt --cc warc` refused `--wb`) + + Python API of the new commands: `cdx_toolkit.indexes.open_index()` with `IndexQuery` (`records()`, `count()`, `explain()`) + and `cdx_toolkit.repackage.repackage()`; `cdxt index` and `cdxt repackage` run the same code + + deprecated `cdx_toolkit.warc.get_writer()` and `CDXToolkitWARCWriter` (FutureWarning): use `repackage()` + + new extras: `athena`, `duckdb`, `hf` (`huggingface_hub>=1.7.2`, the first version whose `HfFileSystem` handles + Storage Buckets), `parquet`; `fsspec` and `aiofiles` are now required + +- 0.9.39 + + `CDXToolkitWARCWriter` (`cdx_toolkit.warc.get_writer()`) supports the context manager protocol (`with ... as writer:`), + so the WARC file is closed, and an S3 upload committed, also on errors; `close()` still works (#70) + + WARC files can be written to S3 (and other fsspec file systems); `cdx_toolkit[s3]` extra (#67) + + settings are read from `CDXT_*` environment variables (`settings.py`), common CLI code moved to `utils.py`, + the CLI and the example close the WARC writer (#67) + + weekly GitHub Action that checks Common Crawl's CDX index server (#60) + + PyPI releases with a manually dispatched trusted-publisher workflow (#69) + - 0.9.38 + deprecated support for py3.7 and py.3.8 + added support for py3.13 and py3.14 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7f991b..81229e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,19 +34,46 @@ make test make unit ``` -By default, all remote requests are mocked. To change this behaviour and actually call remote APIs (if you run this from a whitelisted IP address), the following environment variable can be set: +By default, all remote requests are mocked: the CDX API and HTTP downloads are served from recorded data in `tests/data`, S3 is a local [moto](https://github.com/getmoto/moto) server (installed with `.[all]`; without it, the S3 tests are skipped), and the Hugging Face reads are faked. The mocked requests still wait for the per-host rate limits ([docs/settings.md](docs/settings.md)); the CI shortens them: ```bash -export DISABLE_MOCK_RESPONSES=1 +export CDXT_DEFAULT_MIN_RETRY_INTERVAL=0.01 CDXT_CC_INDEX_MIN_RETRY_INTERVAL=0.01 \ + CDXT_CC_DATA_MIN_RETRY_INTERVAL=0.01 CDXT_IA_MIN_RETRY_INTERVAL=0.01 ``` +To test services live instead, set `CDXT_LIVE_TESTS` to a comma-separated list of them, or `all` (unset the short intervals above, the live servers rate-limit): + +| Service | Live tests | +|---|---| +| `http` | the CDX API (index.commoncrawl.org, web.archive.org) and HTTP downloads (data.commoncrawl.org); needs an allowed IP address | +| `s3` | the S3 tests against AWS, and DuckDB queries of Common Crawl's index on S3; needs AWS credentials | +| `athena` | the Athena queries; needs AWS credentials with Athena access | +| `hf` | `cdxt index cc-url-index-hf`, and `cdxt repackage` reading from `hf://buckets/commoncrawl/commoncrawl` and writing to `hf://buckets/commoncrawl/commoncrawl-ci-temp` (`CDXT_TEST_HF_BUCKET`); writing needs a Hugging Face token (`HF_TOKEN` or `hf auth login`) | + +```bash +CDXT_LIVE_TESTS=s3,athena,hf make test +``` + +The tests that can run live are marked `live` and `live_`, so `-m` selects only them: + +```bash +CDXT_LIVE_TESTS=all pytest -m live tests # all live tests +CDXT_LIVE_TESTS=s3 pytest -m live_s3 tests # only the S3 tests +``` + +The marker only selects the tests; without the service in `CDXT_LIVE_TESTS`, they are skipped (`s3`, `athena`, `hf`) or run with mocked responses (`http`, and `s3` against moto). + +The CI workflow of pull requests runs the mocked tests only; the `CI-slow` workflow (started manually) runs all live tests (`http,s3,athena,hf`) with AWS credentials and `HF_TOKEN`. + +The WARC records that the `cdxt repackage` tests download (from `data.commoncrawl.org` or `s3://commoncrawl`) are recorded in `tests/data/repackage/warc_records.jsonl`, and DuckDB queries a small local index built from the same captures (the `local_url_index` fixture in `tests/conftest.py`). + If the remote APIs change, new mock data can be semi-automatically collected by setting another environment variable, running corresponding unit tests, and overwriting existing mock data in `tests/data/mock_responses`: ```bash -# set environment variable (DISABLE_MOCK_RESPONSES should not be set) -export SAVE_MOCK_RESPONSES=./tmp/mock_responses +# set environment variable (CDXT_LIVE_TESTS should not have http) +export CDXT_SAVE_MOCK_RESPONSES=./tmp/mock_responses -# run the test for what mock data should be saved to $SAVE_MOCK_RESPONSES//.jsonl +# run the test for what mock data should be saved to $CDXT_SAVE_MOCK_RESPONSES//.jsonl pytest tests/test_cli.py::test_basics ``` diff --git a/Makefile b/Makefile index 8a60525..0e5b07a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: init test clean_coverage test_coverage lint distclean distcheck dist install +.PHONY: init test example clean_coverage test_coverage lint distclean distcheck dist install init36: # packages are deprecating support, so this uses exact versions @@ -12,6 +12,9 @@ unit: test: PYTHONPATH=. py.test --doctest-modules cdx_toolkit tests -v -v + +# runs against the live Common Crawl index (not part of `test`: connections from CI runners time out) +example: PYTHONPATH=. examples/iter-and-warc.py clean_coverage: @@ -20,7 +23,7 @@ clean_coverage: test_coverage: clean_coverage # PYTHONPATH=. coverage run -a --source=cdx_toolkit,examples examples/iter-and-warc.py # -rA to see all output, pass or fail - # LOGLEVEL=DEBUG + # CDXT_LOGLEVEL=DEBUG # -vvvv -s PYTHONPATH=. py.test -rA -s --doctest-modules --cov-report=xml --cov-append --cov cdx_toolkit tests -v -v coverage report diff --git a/README.md b/README.md index bab194d..f764212 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # cdx_toolkit -[![build](https://github.com/cocrawler/cdx_toolkit/actions/workflows/ci.yaml/badge.svg)](https://github.com/cocrawler/cdx_toolkit/actions/workflows/ci.yaml) [![coverage](https://codecov.io/gh/cocrawler/cdx_toolkit/graph/badge.svg?token=M1YJB998LE)](https://codecov.io/gh/cocrawler/cdx_toolkit) [![Apache License 2.0](https://img.shields.io/github/license/cocrawler/cdx_toolkit.svg)](LICENSE) +[![build](https://github.com/commoncrawl/cdx_toolkit/actions/workflows/ci.yaml/badge.svg)](https://github.com/commoncrawl/cdx_toolkit/actions/workflows/ci.yaml) [![coverage](https://codecov.io/gh/commoncrawl/cdx_toolkit/graph/badge.svg?token=M1YJB998LE)](https://codecov.io/gh/commoncrawl/cdx_toolkit) [![Apache License 2.0](https://img.shields.io/github/license/commoncrawl/cdx_toolkit.svg)](LICENSE) cdx_toolkit is a set of tools for working with CDX indices of web crawls and archives, including those at the Common Crawl Foundation @@ -27,40 +27,151 @@ or clone this repo and use `pip install .` Optional extras: ``` -$ pip install cdx_toolkit[s3] # enable S3 and other remote filesystem support -$ pip install cdx_toolkit[all] # install all optional dependencies +$ pip install 'cdx_toolkit[s3]' # S3 input/output and WARC downloads from s3:// +$ pip install 'cdx_toolkit[duckdb]' # query the columnar URL index with DuckDB +$ pip install 'cdx_toolkit[athena]' # query the columnar URL index with AWS Athena +$ pip install 'cdx_toolkit[hf]' # Hugging Face Storage Buckets (hf://) +$ pip install 'cdx_toolkit[parquet]' # Parquet output (and input of cdxt repackage) +$ pip install 'cdx_toolkit[all]' # install all optional dependencies ``` +### Settings + +The AWS settings of S3, Athena and DuckDB requests, and the rate limits and retries of HTTP requests to +Common Crawl and the Internet Archive (with their environment variables), are described in +[docs/settings.md](docs/settings.md). + ## Command-line tools +The `cdxt` command line tool has two commands: `index` selects captures +from an index, and `repackage` fetches the selected captures and writes +them into new WARC files. + ``` -$ cdxt --cc size 'commoncrawl.org/*' -$ cdxt --cc --limit 10 iter 'commoncrawl.org/*' # returns the most recent year -$ cdxt --crawl 3 --limit 10 iter 'commoncrawl.org/*' # returns the most recent 3 crawls -$ cdxt --cc --limit 10 --filter '=status:200' iter 'commoncrawl.org/*' +$ cdxt index cc --url 'commoncrawl.org/*' --limit 10 # returns the most recent year +$ cdxt index cc --url 'commoncrawl.org/*' --crawl 3 --limit 10 # returns the most recent 3 crawls +$ cdxt index cc --url 'commoncrawl.org/*' --limit 10 --filter '=status:200' +$ cdxt index cc --domain commoncrawl.org --limit 10 -o captures.csv +$ cdxt index cc --url 'commoncrawl.org/*' --count # estimated number of captures + +$ cdxt index ia --url 'commoncrawl.org/*' --limit 10 # will show the beginning of IA's crawl + +$ cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.csv # DuckDB +$ cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o captures.csv -$ cdxt --ia --limit 10 iter 'commoncrawl.org/*' # will show the beginning of IA's crawl -$ cdxt --ia --limit 10 warc 'commoncrawl.org/*' +$ cdxt index cc --url 'commoncrawl.org/*' --limit 10 -o captures.jsonl +$ cdxt repackage captures.jsonl --prefix COMMONCRAWL +$ cdxt index ia --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --wb ia --prefix IA ``` -cdxt takes a large number of command line switches, controlling -the time period and all other CDX query options. cdxt can generate -WARC, jsonl, and csv outputs. +Usage guides with examples for different setups: + +- [HTTP](docs/usage/http.md): the CDX APIs and Common Crawl's HTTP server, no account needed +- [AWS: Athena and S3](docs/usage/aws-athena-s3.md): the columnar URL index with Athena or DuckDB, and WARC records from `s3://commoncrawl` +- [Hugging Face](docs/usage/huggingface.md): the Storage Bucket `hf://buckets/commoncrawl/commoncrawl`, and HF Jobs + +### cdxt index + +`cdxt index INDEX` takes one of these indexes. How it is queried is detected from INDEX, or set with +`--index-backend cdx-api|cdx-files|duckdb|athena` (e.g. `cdx-files` for a CDX file at an `https://` +URL without file extension): + +| INDEX | Index | +|---|---| +| `cc`, `ia` | Common Crawl or Internet Archive CDX API | +| `https://...` (no file extension) | another CDX API server, e.g. `https://web.archive.org/cdx/search/cdx` | +| `cc-url-index-s3` | Common Crawl's [columnar URL index](https://commoncrawl.org/blog/index-to-warc-files-and-urls-in-columnar-format) on S3 (`s3://commoncrawl/cc-index/table/cc-main/warc/`), queried with DuckDB; needs AWS credentials and `cdx_toolkit[duckdb]` | +| `cc-url-index-hf` | the same index on Hugging Face (`hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/`, recent crawls only), queried with DuckDB; needs `cdx_toolkit[duckdb,hf]`. Much slower than S3 | +| `*.parquet`, or a hive-partitioned directory with `--index-backend duckdb` | another columnar index, local or remote, queried with DuckDB | +| `athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://bucket/path/` | the columnar URL index queried with AWS Athena (database `ccindex`, table `ccindex`; add `&table=...` for another table); needs `cdx_toolkit[athena]` | +| other paths, e.g. `'s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz'` | CDX files (one capture per line, optionally gzipped), local or remote, a single file or a glob | + +Captures are selected with `--url` (a URL pattern like `example.com/*` +or `*.example.com`, or an exact URL), `--hostname` (an exact hostname like +`www.example.com`) and `--domain` (a domain including its subdomains). +These can be repeated and combined; a capture is selected if it +matches any of them. All other options, like `--crawl`, `--from`, +`--to`, `--filter` and `--url-fgrep`, narrow the selection down. +Options an index does not support (e.g. `--filter` with DuckDB, or `--cc-mirror` +and `--cc-sort` with any other index than `cc`) are rejected; `--yes` is +accepted by every index (only the SQL indexes ask for confirmation). +For CDX files and the SQL indexes, `--url` patterns are matched against SURT +keys (`url_surtkey` in the columnar index), like the CDX API does: `www.` is +ignored, so `--url 'example.com/*'` also matches `www.example.com`, while +`--hostname example.com` does not. + +CDX files also support `--url-file PATH`: URL patterns (with the same syntax +as `--url`, e.g. `*.example.com` for a domain) from a file with one pattern per +line, or from the `url` column (or `--url-column NAME`) of a CSV, TSV, JSONL or +Parquet file. For lists of hosts, use `--url-file-type domain`: then an entry +like `example.com` matches the domain and its subdomains, and an entry with a +path like `example.com/reports` all URLs starting with it. The patterns are matched against the SURT keys of the CDX lines, +so long URL lists are fast. For DuckDB and Athena, join the index with a table +of the URLs in a custom `--sql` query instead. + +With a glob of CDX files and an `-o` path without file extension (or ending +with `/`), `cdxt index` writes one output per CDX file, with the same relative +path and the selected lines unchanged (e.g. to filter a whole crawl's index for +a URL list), and `--processes N` filters N files in parallel. No file is +written for CDX files without matches. The directory has to be new or empty, so +that no output of an earlier run is mixed with this one: -If you don't specify much about the crawls or dates or number of -records you're interested in, some default limits will kick in to -prevent overly-large queries. These default limits include a maximum -of 1000 records (`--limit 1000`) and a limit of 1 year of CC indexes. -To exceed these limits, use `--limit` and `--crawl` or `--from` and -`--to`. +``` +$ cdxt index 's3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz' \ + --url-file eot-hosts.txt --url-file-type domain -o s3://my-bucket/eot/CC-MAIN-2024-30/indexes --processes 8 +$ cdxt repackage 's3://my-bucket/eot/CC-MAIN-2024-30/indexes/cdx-*.gz' --prefix EOT-2024 +``` -If it seems like nothing is happening, add `-v` or `-vv` at the start: +An `-o` path with an unknown extension (e.g. `out.txt`) needs `--output-format`. +`cdxt index` refuses an output that would overwrite one of the files the query +reads (the CDX or Parquet files, the `--url-file`). + +The SQL indexes (DuckDB, Athena) also support `--language CODE` (repeatable): +captures whose primary content language (the first of the `languages` field) +is one of the given [ISO 639-3](https://iso639-3.sil.org/) codes, e.g. `eng`, +`deu` or `fra` (three letters; e.g. `en` is rejected). + +The columnar index can also be queried with your own SQL: `--sql query.sql` +(or `--sql -` for stdin) runs the query as it is, instead of `--hostname`, +`--domain`, `--crawl`, `--from`, `--to`, `--language` and `--limit`; the query names the +Parquet files it reads, see [examples/homepages.sql](examples/homepages.sql), and the +[usage guides](docs/usage/) for more. Without `--sql`, the query selects the +CDX fields (`url`, `timestamp`, `status`, `mime`, `digest`, `filename`, +`offset`, `length`, `languages`, ... and `crawl`) for the given hosts and +domains; `--dry-run` prints it without running it. + +SQL queries that are not restricted to at most 10 crawls (with `--crawl`, +or any `--sql` query) ask for confirmation first, because Athena bills per +TB scanned and DuckDB would read a lot of data. Pass `--yes` to skip the +question, e.g. in scripts. + +The captures are written to stdout or to the file given with +`-o/--output` (local or remote, e.g. `s3://...` with `cdx_toolkit[s3]`; +gzipped if the name ends with `.gz`). The output format is JSONL, CDXJ, +CSV, TSV or Parquet (needs `cdx_toolkit[parquet]`, not for stdout), detected +from the file extension or set with `--output-format`; stdout defaults to +JSONL. `cdx` is the same as `cdxj`: Common Crawl's index files have the `.cdx` +extension but use the CDXJ format (crawl index JSON). Use `--fields url,status,timestamp` to select fields; by default all +fields are written. + +`--count` prints the number of captures instead. For the CDX API this is +a cheap estimate based on the number of index pages (for `--hostname`, of the +host with and without `www.`, which the CDX API does not distinguish; the +captures of the other one are dropped when they are read); for CDX files it is +an exact count, and for SQL indexes an exact `count(*)`. + +If you don't specify the crawls or dates you're interested in, queries +of `cc` are limited to the CC indexes of the most recent year, to +prevent overly-large queries. To exceed this limit, use `--crawl` or +`--from` and `--to`. There is no default `--limit`. + +If it seems like nothing is happening, add `-v` or `-vv`: ``` -$ cdxt -vv --cc size 'commoncrawl.org/*' +$ cdxt -vv index cc --url 'commoncrawl.org/*' --count ``` -## Selecting particular CCF crawls +#### Selecting particular CCF crawls Common Crawl's data is divided into "crawls", which were yearly at the start, and are currently done monthly. There are over 100 of them. @@ -71,7 +182,7 @@ covers all of these crawls -- we have 1 index per crawl. The way you ask for a particular crawl is: ``` -$ cdxt --crawl CC-MAIN-2024-33 iter 'commoncrawl.org/*' +$ cdxt index cc --crawl CC-MAIN-2024-33 --url 'commoncrawl.org/*' ``` - `--crawl CC-MAIN-2024-33` is a single crawl. @@ -86,42 +197,188 @@ this index at [the blog post about it](https://commoncrawl.org/blog/index-to-war The Internet Archive cdx index is organized as a single crawl that goes from the very beginning until now. That's why there is no `--crawl` for -`--ia`. Note that cdx queries to `--ia` will default to one year year -and limit 1000 entries if you do not specify `--from`, `--to`, and `--limit`. +`ia`, and no default time range: without `--from` and `--to`, a query of +`ia` starts at the beginning of IA's crawl. -## Selecting by time +#### Selecting by time In most cases you'll probably use --crawl to select the time range for Common Crawl queries, but for the Internet Archive you'll need to specify a time range like this: ``` -$ cdxt --ia --limit 1 --from 2008 --to 200906302359 iter 'commoncrawl.org/*' +$ cdxt index ia --limit 1 --from 2008 --to 200906302359 --url 'commoncrawl.org/*' ``` In this example the time range starts at the beginning of 2008 and ends on June 30, 2009 at 23:59. All times are in UTC. If you do not -specify a time range (and also don't use `--crawl`), you'll get the -most recent year. +specify a time range (and also don't use `--crawl`), `cc` queries get the +most recent year, and `ia` queries all years. + +#### Filter syntax + +Filters can be used to limit captures to a subset of the results. + +Any field name in the output of `cdxt index` can be used in a +filter. These field names are appropriately renamed if the source is +'ia'. The different syntax of filter modifiers for 'ia' and 'cc' is +not fully abstracted away by cdx_toolkit. + +The basic syntax of a filter is `[modifier]field:expression`, for +example `=status:200` or `!=status:200`. + +'cc'-style filters (pywb) come in six flavors: substring match, exact +string, full-match regex, and their inversions. These are indicated by +a modifier of nothing, '=', '\~', '!', '!=', and '!\~', respectively. -## The full syntax for command-line tools +'ia'-style filters (Wayback/OpenWayback) come in two flavors, a full-match +regex and an inverted full-match regex: 'status:200' and '!status:200' + +Multiple filters will be combined with AND. For example, to limit +captures to those which do not have status 200 and do not have status 404, ``` -$ cdxt --help -$ cdxt iter --help -$ cdxt warc --help -$ cdxt size --help +$ cdxt index cc --filter '!=status:200' --filter '!=status:404' ... ``` -for full details. Note that argument order really matters; each switch -is valid only either before or after the {iter,warc,size} command. +Note that filters that discard large numbers of captures put a high +load on the CDX server -- for example, a filter that returns just a +few captures from a domain that has tens of millions of captures is +likely to run very slowly and annoy the owner of the CDX server. + +See https://github.com/webrecorder/pywb/wiki/CDX-Server-API#filter (pywb) +and https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server#filtering (wayback) +for full details of filter modifiers. + +### cdxt repackage + +`cdxt repackage INDEX_PATH` reads captures written by `cdxt index` (a +local or remote file, a glob like `s3://bucket/indexes/*.cdx.gz`, or stdin +if no path or `-` is given) and writes them into new WARC files. The input +format (JSONL, CDXJ, CSV or TSV) is detected from the first line, so Common +Crawl's CDX files can be used directly; `*.parquet` files are read as Parquet. +Use `--input-format` to set the format instead. +The columnar index column names (`warc_filename`, `warc_record_offset`, +`warc_record_length`) work too. A `filename` (or `warc_filename`) that is a +full URL (`s3://...`, `https://...`, `hf://...`) is used as it is, otherwise +it is relative to `--warc-download-prefix`. + +Captures with a WARC location (the `filename`, `offset` and `length` +fields, e.g. all Common Crawl captures) are copied byte-for-byte from +`--warc-download-prefix`: `https://data.commoncrawl.org` (default), +`s3://commoncrawl` (fastest in AWS us-east-1, needs `cdx_toolkit[s3]`) or +`hf://buckets/commoncrawl/commoncrawl` (needs `cdx_toolkit[hf]`). They are +written to `[-]-001.warc.gz`, `-002`, ..., where the +prefix can be a local path, `s3://...` or `hf://buckets/...` (default: +`REPACKAGE` in the current directory, e.g. `--prefix out/CC-HOMEPAGES` writes +`out/CC-HOMEPAGES-001.warc.gz`). No file is written when there are no +captures. The captures are +sorted by WARC file and offset first (for faster reads); `--no-sort` keeps +the input order and streams large inputs without keeping them in memory. + +Reads from `hf://` count against the Hugging Face Hub's +[Resolvers rate limit](https://huggingface.co/docs/hub/en/rate-limits) with one +request per WARC file (the records themselves are read from the CDN), so set +`HF_TOKEN` (anonymous access has the lowest limit). + +For large jobs, use `--parallel-readers N` (concurrent reads per process) +and `--processes N` (e.g. the number of vCPUs). With several processes, +the captures are split by WARC file across the processes, which write into +one shared `-001`, `-002`, ... series (which process writes which file +depends on timing; each file is a complete WARC). + +Captures without a WARC location, like Internet Archive captures, are +replayed from the wayback machine given with `--wb` (`ia` is a shortcut for +`https://web.archive.org/web`) and written to the same +`[-]-001.warc.gz`, ... series, also with `--parallel-readers` and +`--add-metadata-record` (`--processes` cannot be used with `--wb`: the requests to the wayback machine +are rate-limited per process). A replay needs the `url` (http or https), `timestamp` and `status` of +each capture. Common Crawl captures (`filename` `crawl-data/...`) are refused with `--wb`: they are +copied from their WARC files, while a replay would get another archive's capture. + +In both cases, if WARC files with the same prefix and subprefix exist already +(e.g. from an earlier run), `cdxt repackage` stops before fetching (exit +status 2) instead of overwriting them or adding to them: use another +`--prefix` or `--subprefix`. + +Use `--size` (e.g. `100MB` or `1GiB`) to set the target size of each WARC +file, `--creator`, `--operator`, `--is-part-of` and `--description` to fill +in the `warcinfo` record, and `--add-metadata-record FILE` to add a file +(e.g. the URL list of the job) as a metadata record to each WARC file. + +A capture that cannot be fetched (after retries) does not stop the job: the +other captures are still written, and at the end `cdxt repackage` logs how +many captures were requested, written and failed, and exits with status 1. +A failure to write the output stops the job at once, with `--processes` all +processes; the WARC files being written are removed (or their S3 uploads +aborted), the files before them are complete. A process that has not stopped +after 60 seconds is terminated, which can leave an incomplete S3 multipart +upload (an S3 lifecycle rule for incomplete uploads removes it). + +### Migrating from the old commands + +The command lines of cdx_toolkit 0.9.39 (`cdxt --cc iter ...`, `size`, +`warc`) still work, but are deprecated and will be removed in a future +version; they print a suggested new command line. The old `warc` now writes +`TEST-001.warc.gz` and exits with status 1 if captures could not be fetched. +See the [migration guide](docs/cli-migration-guide.md) for all changes and +more examples. + +| Old | New | +|---|---| +| `cdxt --cc --limit 10 iter 'example.com/*'` | `cdxt index cc --url 'example.com/*' --limit 10` | +| `cdxt --crawl 3 iter 'example.com/*'` | `cdxt index cc --url 'example.com/*' --crawl 3` | +| `cdxt --source URL iter 'example.com/*'` | `cdxt index URL --url 'example.com/*'` | +| `cdxt --cc size 'example.com/*'` | `cdxt index cc --url 'example.com/*' --count` | +| `cdxt --ia warc 'example.com/*'` | `cdxt index ia --url 'example.com/*' \| cdxt repackage --wb ia` | + +All options now go after the command. `iter --csv`/`--jsonl` became +`--output-format`, all fields are written by default (`--all-fields` is +gone), `--get` is no longer needed (`--closest` uses a single query), and +`--url-fgrep`/`--url-fgrepv` moved from `warc` to `index`. + + +### The full syntax for command-line tools + +``` +$ cdxt --help +$ cdxt index --help +$ cdxt repackage --help +``` -Add -v (or -vv) to see what's going on under the hood. +for full details. Add -v (or -vv) to see what's going on under the hood. ## Python programming example Everything that you can do on the command line, and much more, can -be done by writing a Python program. +be done by writing a Python program. `cdxt index` and `cdxt repackage` +run the same code as `open_index()` with an `IndexQuery`, and `repackage()`: + +``` +from cdx_toolkit.indexes import IndexQuery, open_index +from cdx_toolkit.repackage import repackage + +index = open_index('cc') # any INDEX of `cdxt index`, e.g. 'cc-url-index-s3' or 'athena://...' +query = IndexQuery(urls='commoncrawl.org/*', crawl='CC-MAIN-2024-33', limit=10) + +print(index.count(query)) # like --count +print(index.explain(query)) # like --dry-run +for capture in index.records(query): + print(capture['url'], capture['timestamp']) + +result = repackage(index.records(query), prefix='out/COMMONCRAWL', size=100_000_000) +print(result.written, 'written,', result.failed, 'failed') +``` + +The fields of `IndexQuery` are the options of `cdxt index` (`urls`, `hostnames`, `domains`, +`url_file`, `crawl`, `from_ts`, `to`, `filters`, `languages`, `url_fgrep`, `limit`, `sql`, ...), and the +arguments of `repackage()` those of `cdxt repackage` (`subprefix`, `size`, `warc_download_prefix`, +`wb`, `processes`, `parallel_readers`, `metadata_records`, ...). `repackage()` also takes the path of +an index file. To write the captures to a file like `cdxt index -o`, use +`cdx_toolkit.record_io.write_records(index.records(query), 'captures.csv', 'csv', fixed_schema=index.fixed_schema)`. + +`CDXFetcher` is the lower-level client of the CDX API, which can also +fetch the content of single captures: ``` import cdx_toolkit @@ -148,43 +405,10 @@ You can also fetch the content of the web capture as bytes: print(obj.content) ``` -There's a full example of iterating and selecting a subset of captures -to write into an extracted WARC file in [examples/iter-and-warc.py](examples/iter-and-warc.py) - -## Filter syntax - -Filters can be used to limit captures to a subset of the results. - -Any field name listed in `cdxt iter --all-fields` can be used in a -filter. These field names are appropriately renamed if the source is -'ia'. The different syntax of filter modifiers for 'ia' and 'cc' is -not fully abstracted away by cdx_toolkit. +There's a full example of selecting a subset of captures and writing them +into a WARC file in [examples/iter-and-warc.py](examples/iter-and-warc.py). +(`cdx_toolkit.warc.get_writer()` is deprecated: use `repackage()`.) -The basic syntax of a filter is `[modifier]field:expression`, for -example `=status:200` or `!=status:200`. - -'cc'-style filters (pywb) come in six flavors: substring match, exact -string, full-match regex, and their inversions. These are indicated by -a modifier of nothing, '=', '\~', '!', '!=', and '!\~', respectively. - -'ia'-style filters (Wayback/OpenWayback) come in two flavors, a full-match -regex and an inverted full-match regex: 'status:200' and '!status:200' - -Multiple filters will be combined with AND. For example, to limit -captures to those which do not have status 200 and do not have status 404, - -``` -$ cdxt --cc --filter '!=status:200' --filter '!=status:404' iter ... -``` - -Note that filters that discard large numbers of captures put a high -load on the CDX server -- for example, a filter that returns just a -few captures from a domain that has tens of millions of captures is -likely to run very slowly and annoy the owner of the CDX server. - -See https://github.com/webrecorder/pywb/wiki/CDX-Server-API#filter (pywb) -and https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server#filtering (wayback) -for full details of filter modifiers. ## CDX Jargon, Field Names, and such @@ -219,8 +443,8 @@ should be present in all records, if you want to know exactly what it is. The **limit** argument limits how many captures will be returned. To -help users not shoot themselves in the foot, a limit of 1,000 is -applied to --get and .get() calls. +help users not shoot themselves in the foot, a default limit of 1,000 is +applied to `CDXFetcher.get()` calls. A **filter** allows a user to select a subset of CDX records, reducing network traffic between the CDX API server and the user. For example, @@ -234,8 +458,8 @@ status of '-', not 200. CDX API servers support a **paged interface** for efficient access to large sets of URLs. cdx_toolkit iterators always use the paged interface. cdx_toolkit is also polite to CDX servers by being single-threaded and -serial. If it's not fast enough for you, consider downloading Common -Crawl's index files directly. +serial. If it's not fast enough for you, consider querying Common +Crawl's index files directly with `cdxt index` (CDX files or the columnar index). A **digest** is a sha1 checksum of the contents of a capture. The purpose of a digest is to be able to easily figure out if 2 captures @@ -270,11 +494,9 @@ figuring out the charset using an algorithm similar to browsers. WARC generation should do smart(er) things with revisit records. -Right now the CC code selects which monthly CC indices to use based -solely on date ranges. It would be nice to have an alternative so that -a client could iterate against the most recent N CC indices, and -also have the default one-year lookback use an entire monthly index -instead of a partial one. +Without `--crawl`, the CC code selects which monthly CC indices to use +based solely on date ranges. It would be nice to have the default +one-year lookback use an entire monthly index instead of a partial one. ## Status diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ee56d4a..05173d7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ jobs: python.version: '3.6' Python36debug: python.version: '3.6' - myenv: LOGLEVEL=DEBUG EXACT_PIP_VERSIONS=1 + myenv: CDXT_LOGLEVEL=DEBUG EXACT_PIP_VERSIONS=1 Python37: python.version: '3.7' Python38: diff --git a/cdx_toolkit/__init__.py b/cdx_toolkit/__init__.py index efa2d9f..896fb5c 100644 --- a/cdx_toolkit/__init__.py +++ b/cdx_toolkit/__init__.py @@ -84,7 +84,7 @@ def cdx_to_captures(resp, wb=None, warc_download_prefix=None): ret = munge_fields(fields, lines) return [CaptureObject(r, wb=wb, warc_download_prefix=warc_download_prefix) for r in ret] - raise ValueError('cannot decode response, first bytes are '+repr(text[:50])) # pragma: no cover + raise ValueError(f'cannot decode response (status {resp.status_code}), first bytes are {text[:50]!r}') class CaptureObject(MutableMapping): diff --git a/cdx_toolkit/aws.py b/cdx_toolkit/aws.py new file mode 100644 index 0000000..1bb4797 --- /dev/null +++ b/cdx_toolkit/aws.py @@ -0,0 +1,211 @@ +''' +Defaults for cdxt's requests to S3, as recommended by Common Crawl for downloading from s3://commoncrawl: + + region = us-east-1 + retry_mode = adaptive + max_attempts = 100000 + s3 = + addressing_style = virtual + payload_signing_enabled = false + max_concurrent_requests = 2 + (multipart_threshold = multipart_chunksize = 10GB: no multipart transfers) + +The user stays in control: a default is only used for a setting that is not configured in the +environment (AWS_REGION / AWS_DEFAULT_REGION, AWS_RETRY_MODE, AWS_MAX_ATTEMPTS), in the AWS config +file of the active profile (~/.aws/config, AWS_PROFILE) or explicitly (e.g. the retries in the fsspec +configuration). CDXT_AWS_DEFAULTS=0 disables all defaults. The region is, in this order: an explicit +region, AWS_REGION (which botocore itself ignores), AWS_DEFAULT_REGION, the profile, the default. + +The multipart settings are for whole-file transfers (`aws s3 cp`): cdxt reads WARC records with one +range request each and reads index files sequentially, so it makes no multipart downloads anyway. + +The same settings are used for the Athena client and, as far as it has equivalents, for DuckDB. +''' +import functools +import logging +import os +import re +from typing import List, Optional + + +LOGGER = logging.getLogger(__name__) + +DEFAULT_REGION = 'us-east-1' +DEFAULT_RETRY_MODE = 'adaptive' +DEFAULT_MAX_ATTEMPTS = 100000 # total attempts, as `max_attempts` in the AWS config file +DEFAULT_S3 = {'addressing_style': 'virtual', 'payload_signing_enabled': False} +DEFAULT_MAX_CONCURRENT_REQUESTS = 2 + +# environment variables that configure a setting (botocore reads them itself, except AWS_REGION) +SETTING_ENV_VARS = { + 'region': ('AWS_REGION', 'AWS_DEFAULT_REGION'), + 'retry_mode': ('AWS_RETRY_MODE',), + 'max_attempts': ('AWS_MAX_ATTEMPTS',), +} + + +def defaults_enabled() -> bool: + return os.environ.get('CDXT_AWS_DEFAULTS', '1').strip().lower() not in ('0', 'false', 'no', 'off') + + +@functools.lru_cache(maxsize=None) +def user_aws_config() -> dict: + """The AWS config file settings of the active profile ({} without botocore or a config file).""" + try: + import botocore.session + + return dict(botocore.session.Session().get_scoped_config()) + except Exception as e: # botocore missing, unknown profile, unreadable config file + LOGGER.debug('no AWS config file settings: %r', e) + return {} + + +def user_configured(setting: str) -> bool: + if any(os.environ.get(name) for name in SETTING_ENV_VARS.get(setting, ())): + return True + return setting in user_aws_config() + + +def user_s3_setting(name: str): + s3 = user_aws_config().get('s3') + return s3.get(name) if isinstance(s3, dict) else None + + +# the retry settings that bound the number of attempts (botocore prefers total_max_attempts) +RETRY_BOUNDS = ('max_attempts', 'total_max_attempts') + + +def botocore_config_kwargs() -> dict: + """kwargs for botocore.config.Config with the defaults for the settings the user did not configure. + + botocore merges the `retries` and `s3` settings key by key with the environment and the AWS + config file, so the settings the user configured keep their values.""" + kwargs = {} + if os.environ.get('AWS_REGION'): + kwargs['region_name'] = os.environ['AWS_REGION'] # botocore only reads AWS_DEFAULT_REGION + if not defaults_enabled(): + return kwargs + if not user_configured('region'): + kwargs['region_name'] = DEFAULT_REGION + retries = {} + if not user_configured('retry_mode'): + retries['mode'] = DEFAULT_RETRY_MODE + if not user_configured('max_attempts'): + retries['total_max_attempts'] = DEFAULT_MAX_ATTEMPTS + if retries: + kwargs['retries'] = retries + s3 = {name: value for name, value in DEFAULT_S3.items() if user_s3_setting(name) is None} + if s3: + kwargs['s3'] = s3 + return kwargs + + +def merge_config_kwargs(defaults: dict, explicit: dict) -> dict: + """Explicit botocore config kwargs win over the defaults; the `retries` and `s3` dicts are merged key by key, + and an explicit bound of the attempts (max_attempts or total_max_attempts) replaces the default one.""" + merged = dict(defaults) + for key, value in explicit.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + default = dict(merged[key]) + if key == 'retries' and any(bound in value for bound in RETRY_BOUNDS): + for bound in RETRY_BOUNDS: + default.pop(bound, None) + merged[key] = {**default, **value} + else: + merged[key] = value + return merged + + +def botocore_config(region_name: Optional[str] = None, **kwargs): + """A botocore Config with the defaults, plus the given options, e.g. timeouts (they win over the defaults). + + An explicit `region_name` wins over the default and the user's configuration.""" + from botocore.config import Config + + config_kwargs = merge_config_kwargs(botocore_config_kwargs(), kwargs) + if region_name: + config_kwargs['region_name'] = region_name + return Config(**config_kwargs) + + +def max_concurrent_requests() -> int: + configured = user_s3_setting('max_concurrent_requests') + if configured is not None: + return int(configured) + return DEFAULT_MAX_CONCURRENT_REQUESTS + + +def configure_fsspec_s3() -> None: + """Use the defaults for s3:// paths opened with fsspec (s3fs), without changing options that are already + set in the fsspec configuration (~/.config/fsspec/, FSSPEC_S3_* environment variables).""" + if not defaults_enabled(): + return + import fsspec.config + + conf = fsspec.config.conf.setdefault('s3', {}) + config_kwargs = merge_config_kwargs(botocore_config_kwargs(), dict(conf.get('config_kwargs') or {})) + if config_kwargs: + conf['config_kwargs'] = config_kwargs + conf.setdefault('max_concurrency', max_concurrent_requests()) + + +def process_pool(max_workers: int, mp_context=None, initializer=None, initargs=()): + """A ProcessPoolExecutor whose processes use the S3 defaults of configure_fsspec_s3() like this one: a process + started with spawn (macOS, Windows) does not inherit the changes of fsspec.config.conf. Settings from + files and environment variables apply in every process; other in-memory changes of the configuration + are not passed on.""" + from concurrent.futures import ProcessPoolExecutor + + return ProcessPoolExecutor( + max_workers=max_workers, mp_context=mp_context, initializer=_init_process, initargs=(initializer, initargs) + ) + + +def _init_process(initializer, initargs) -> None: + configure_fsspec_s3() + if initializer is not None: + initializer(*initargs) + + +def _user_value(setting: str) -> Optional[str]: + for name in SETTING_ENV_VARS.get(setting, ()): + if os.environ.get(name): + return os.environ[name] + value = user_aws_config().get(setting) + return str(value) if value is not None else None + + +def duckdb_settings(region_name: Optional[str] = None) -> List[str]: + """SET statements with the defaults for DuckDB's S3 access (httpfs), as far as DuckDB has equivalents. + + Settings the user configured (environment, AWS config file) are used instead of the defaults: + - region -> s3_region (an explicit `region_name` wins) + - max_attempts -> http_retries (attempts - 1), with a constant wait of 1s between retries, because + DuckDB's exponential backoff has no maximum wait + - s3 addressing_style -> s3_url_style (virtual: vhost, path: path) + There is no equivalent of retry_mode (DuckDB has no adaptive mode) and max_concurrent_requests + (only `threads`, which limits all of DuckDB's parallelism).""" + if not defaults_enabled(): + region = region_name or _user_value('region') + return [f"SET s3_region='{region}';"] if region and re.fullmatch(r'[a-z0-9-]+', region) else [] + region = region_name or _user_value('region') or DEFAULT_REGION + if not re.fullmatch(r'[a-z0-9-]+', region): + raise ValueError(f'invalid AWS region {region!r}') + attempts = _user_value('max_attempts') + attempts = int(attempts) if attempts and attempts.isdigit() else DEFAULT_MAX_ATTEMPTS + url_style = 'path' if (user_s3_setting('addressing_style') or DEFAULT_S3['addressing_style']) == 'path' else 'vhost' + return [ + f"SET s3_region='{region}';", + f"SET s3_url_style='{url_style}';", + f'SET http_retries={max(attempts - 1, 0)};', + 'SET http_retry_wait_ms=1000;', + 'SET http_retry_backoff=1;', + ] + + +def aws_region(region_name: Optional[str] = None) -> str: + """The region for AWS clients that need a concrete value (explicit, the user's, or the default).""" + region = region_name or _user_value('region') or DEFAULT_REGION + if not re.fullmatch(r'[a-z0-9-]+', region): + raise ValueError(f'invalid AWS region {region!r}') + return region diff --git a/cdx_toolkit/cli.py b/cdx_toolkit/cli.py deleted file mode 100644 index 1e59e04..0000000 --- a/cdx_toolkit/cli.py +++ /dev/null @@ -1,245 +0,0 @@ -from argparse import ArgumentParser, Namespace -import logging -import csv -import sys -import json -import os - -import cdx_toolkit - -from cdx_toolkit.utils import get_version, setup_cdx_fetcher_and_kwargs - - -LOGGER = logging.getLogger(__name__) - - -def main(args=None): - parser = ArgumentParser(description='cdx_toolkit iterator command line tool') - - parser.add_argument('--version', '-V', action='version', version=get_version()) - parser.add_argument( - '--verbose', - '-v', - action='count', - help='set logging level to INFO (-v) or DEBUG (-vv)' - ) - parser.add_argument( - '--cc', - action='store_const', - const='cc', - help='direct the query to the Common Crawl CDX/WARCs' - ) - parser.add_argument( - '--crawl', - action='store', - help=('crawl names (comma separated) or an integer for the most recent N crawls. ' - 'Implies --cc') - ) - parser.add_argument( - '--ia', - action='store_const', - const='ia', - help='direct the query to the Internet Archive CDX/wayback' - ) - parser.add_argument('--source', action='store', help='direct the query to this CDX server') - parser.add_argument('--wb', action='store', help='direct replays for content to this wayback') - parser.add_argument('--limit', type=int, action='store') - parser.add_argument('--cc-mirror', action='store', help='use this Common Crawl index mirror') - parser.add_argument('--cc-sort', action='store', help='default mixed, alternatively: ascending') - parser.add_argument('--from', action='store') - parser.add_argument('--to', action='store') - parser.add_argument('--filter', action='append', help='see CDX API documentation for usage') - parser.add_argument( - '--get', - action='store_true', - help='use a single get instead of a paged iteration. default limit=1000' - ) - parser.add_argument( - '--closest', - action='store', - help='get the closest capture to this timestamp. use with --get' - ) - - subparsers = parser.add_subparsers(dest='cmd') - subparsers.required = True - - iterate = subparsers.add_parser('iter', help='iterate printing captures') - iterate.add_argument('--all-fields', action='store_true') - iterate.add_argument( - '--fields', - action='store', - default='url,status,timestamp', - help='try --all-fields if you need the list' - ) - iterate.add_argument('--jsonl', action='store_true') - iterate.add_argument('--csv', action='store_true') - iterate.add_argument('url') - iterate.set_defaults(func=iterator) - - warc = subparsers.add_parser('warc', help='iterate over capture content, creating a warc') - warc.add_argument('--prefix', default='TEST', help='prefix for the warc filename') - warc.add_argument( - '--subprefix', - type=str, - default=None, - help='subprefix for the warc filename, default None' - ) - warc.add_argument( - '--size', - type=int, - default=1000000000, - help='target for the warc filesize in bytes' - ) - warc.add_argument( - '--creator', - action='store', - help='creator of the warc: person, organization, service' - ) - warc.add_argument( - '--operator', - action='store', - help='a person, if the creator is an organization' - ) - warc.add_argument( - '--url-fgrep', - action='store', - help='this pattern must be present to warc an url' - ) - warc.add_argument( - '--url-fgrepv', - action='store', - help='this pattern must not be present to warc an url, e.g. /robots.txt' - ) - warc.add_argument( - '--warc-download-prefix', - action='store', - help='prefix for downloading content, automatically set for CC' - ) - warc.add_argument('url') - warc.set_defaults(func=warcer) - - size = subparsers.add_parser('size', help='imprecise count of how many results are available') - size.add_argument('--details', action='store_true', help='show details of each subindex') - size.add_argument('url') - size.set_defaults(func=sizer) - - if args is not None: - cmdline = ' '.join(args) - else: # pragma: no cover - # there's something magic about args and console_scripts - # this fallback is needed when installed by setuptools - if len(sys.argv) > 1: - cmdline = 'cdxt ' + ' '.join(sys.argv[1:]) - else: - cmdline = 'cdxt' - cmd = parser.parse_args(args=args) - set_loglevel(cmd) - cmd.func(cmd, cmdline) - - -def set_loglevel(cmd: Namespace): - loglevel = os.getenv('LOGLEVEL') or 'WARNING' - if cmd.verbose: - if cmd.verbose > 0: - loglevel = 'INFO' - if cmd.verbose > 1: - loglevel = 'DEBUG' - - # because pytest has already initialized the logger, we have to set the root logger - logging.getLogger().setLevel(loglevel) - # for the normal case - logging.basicConfig(level=loglevel) - - LOGGER.info('set loglevel to %s', str(loglevel)) - - -def winnow_fields(cmd: Namespace, fields, obj): - if cmd.all_fields: - printme = obj - else: - printme = dict([(k, obj[k]) for k in fields if k in obj]) - return printme - - -def print_line(cmd: Namespace, writer, printme): - if cmd.jsonl: - print(json.dumps(printme, sort_keys=True)) - elif writer: - writer.writerow(printme) - else: - print(', '.join([' '.join((k, printme[k])) for k in sorted(printme.keys())])) - - -def iterator(cmd: Namespace, cmdline): - cdx, kwargs = setup_cdx_fetcher_and_kwargs(cmd) - fields = set(cmd.fields.split(',')) - if cmd.csv: - writer = csv.DictWriter(sys.stdout, fieldnames=sorted(list(fields))) - writer.writeheader() - else: - writer = None - - if cmd.get: - objs = cdx.get(cmd.url, **kwargs) - for obj in objs: - printme = winnow_fields(cmd, fields, obj) - print_line(cmd, writer, printme) - return - - for obj in cdx.iter(cmd.url, **kwargs): - printme = winnow_fields(cmd, fields, obj) - print_line(cmd, writer, printme) - - -def warcer(cmd: Namespace, cmdline: str): - cdx, kwargs = setup_cdx_fetcher_and_kwargs(cmd) - - ispartof = cmd.prefix - if cmd.subprefix: - ispartof += '-' + cmd.subprefix - - info = { - 'software': 'pypi_cdx_toolkit/'+get_version(), - 'isPartOf': ispartof, - 'description': 'warc extraction generated with: '+cmdline, - 'format': 'WARC file version 1.0', # todo: if we directly read a warc, have this match the warc - } - if cmd.creator: - info['creator'] = cmd.creator - if cmd.operator: - info['operator'] = cmd.operator - - kwargs_writer = {} - if 'size' in kwargs: - kwargs_writer['size'] = kwargs['size'] - del kwargs['size'] - - with cdx_toolkit.warc.get_writer(cmd.prefix, cmd.subprefix, info, **kwargs_writer) as writer: - for obj in cdx.iter(cmd.url, **kwargs): - url = obj['url'] - if cmd.url_fgrep and cmd.url_fgrep not in url: - LOGGER.debug('not warcing due to fgrep: %s', url) - continue - if cmd.url_fgrepv and cmd.url_fgrepv in url: - LOGGER.debug('not warcing due to fgrepv: %s', url) - continue - timestamp = obj['timestamp'] - try: - record = obj.fetch_warc_record() - except RuntimeError: # pragma: no cover - LOGGER.warning('skipping capture for RuntimeError 404: %s %s', url, timestamp) - continue - if obj.is_revisit(): - LOGGER.warning('revisit record being resolved for url %s %s', url, timestamp) - writer.write_record(record) - - -def sizer(cmd: Namespace, cmdline): - cdx, kwargs = setup_cdx_fetcher_and_kwargs(cmd) - - size = cdx.get_size_estimate(cmd.url, **kwargs) - print(size) - - -if __name__ == "__main__": - main() diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py new file mode 100644 index 0000000..c4e8aef --- /dev/null +++ b/cdx_toolkit/cli/__init__.py @@ -0,0 +1,51 @@ +from argparse import ArgumentParser +import sys + +from cdx_toolkit.aws import configure_fsspec_s3 +from cdx_toolkit.hf import check_huggingface_hub, uses_hf +from cdx_toolkit.indexes import UsageError +from cdx_toolkit.utils import get_version + +from . import index, repackage +from .common import IncompleteError, add_verbose_arg, set_loglevel +from .legacy import build_legacy_parser, deprecation_message, is_legacy_command_line + + +def build_parser() -> ArgumentParser: + parser = ArgumentParser(prog='cdxt', description='cdx_toolkit command line tool') + parser.add_argument('--version', '-V', action='version', version=get_version()) + add_verbose_arg(parser) + + subparsers = parser.add_subparsers(dest='cmd', metavar='COMMAND') + subparsers.required = True + index.add_parser(subparsers) + repackage.add_parser(subparsers) + return parser + + +def main(args=None): + if args is None: # pragma: no cover + args = sys.argv[1:] + + # the command lines of cdx_toolkit 0.9.39 (deprecated): `cdxt --cc iter URL`, `size`, `warc` + legacy = is_legacy_command_line(args) + parser = build_legacy_parser() if legacy else build_parser() + cmd = parser.parse_args(args=args) + set_loglevel(cmd.verbose) + if legacy: + sys.stderr.write(deprecation_message(cmd)) + configure_fsspec_s3() # Common Crawl's S3 defaults for s3:// paths, unless configured by the user + cmdline = ' '.join(['cdxt'] + list(args)) + if uses_hf(args): + try: + check_huggingface_hub() # before any work, not when the first hf:// path is opened + except RuntimeError as e: + parser.exit(2, f'cdxt {cmd.cmd}: error: {e}\n') + try: + cmd.func(cmd, cmdline) + except (UsageError, FileNotFoundError) as e: + parser.exit(2, f'cdxt {cmd.cmd}: error: {e}\n') + except IncompleteError as e: + parser.exit(1, f'cdxt {cmd.cmd}: error: {e}\n') + except KeyboardInterrupt: + parser.exit(130, f'cdxt {cmd.cmd}: interrupted\n') diff --git a/cdx_toolkit/cli/__main__.py b/cdx_toolkit/cli/__main__.py new file mode 100644 index 0000000..8273c4f --- /dev/null +++ b/cdx_toolkit/cli/__main__.py @@ -0,0 +1,3 @@ +from . import main + +main() diff --git a/cdx_toolkit/cli/common.py b/cdx_toolkit/cli/common.py new file mode 100644 index 0000000..c0860b2 --- /dev/null +++ b/cdx_toolkit/cli/common.py @@ -0,0 +1,71 @@ +from argparse import SUPPRESS, ArgumentParser, ArgumentTypeError +import logging +import os +import re + + +LOGGER = logging.getLogger(__name__) + + +class IncompleteError(RuntimeError): + """The command ran, but did not do all of its work, e.g. captures could not be fetched (exit status 1).""" + + +SIZE_UNITS = { + '': 1, + 'b': 1, + 'k': 1000, 'kb': 1000, 'kib': 1024, + 'm': 1000 ** 2, 'mb': 1000 ** 2, 'mib': 1024 ** 2, + 'g': 1000 ** 3, 'gb': 1000 ** 3, 'gib': 1024 ** 3, + 't': 1000 ** 4, 'tb': 1000 ** 4, 'tib': 1024 ** 4, +} + + +def add_verbose_arg(parser: ArgumentParser, subcommand: bool = False): + # On subcommands the default is suppressed, so it does not override a -v given before the subcommand. + parser.add_argument( + '--verbose', + '-v', + action='count', + default=SUPPRESS if subcommand else 0, + help='set logging level to INFO (-v) or DEBUG (-vv)', + ) + + +def set_loglevel(verbose: int): + loglevel = os.getenv('CDXT_LOGLEVEL') or os.getenv('LOGLEVEL') or 'WARNING' # LOGLEVEL: the old name + if verbose and verbose > 0: + loglevel = 'INFO' + if verbose and verbose > 1: + loglevel = 'DEBUG' + + # because pytest has already initialized the logger, we have to set the root logger + logging.getLogger().setLevel(loglevel) + # for the normal case + logging.basicConfig(level=loglevel) + + LOGGER.info('set loglevel to %s', str(loglevel)) + + +def parse_size(value: str) -> int: + """Parse a size in bytes like 1000000, 100MB, 1.5GB or 1GiB (argparse type).""" + match = re.fullmatch(r'\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]*)\s*', value) + if not match or match.group(2).lower() not in SIZE_UNITS: + raise ArgumentTypeError(f'invalid size {value!r}, use e.g. 1000000, 100MB or 1GiB') + return int(float(match.group(1)) * SIZE_UNITS[match.group(2).lower()]) + + +def positive_int(value: str) -> int: + """An integer >= 1 (argparse type), e.g. a number of processes.""" + try: + number = int(value) + except ValueError: + number = 0 + if number < 1: + raise ArgumentTypeError(f'invalid value {value!r}, use a positive integer') + return number + + +def parse_fields(value: str): + """Comma-separated list of field names (argparse type).""" + return [f.strip() for f in value.split(',') if f.strip()] diff --git a/cdx_toolkit/cli/index.py b/cdx_toolkit/cli/index.py new file mode 100644 index 0000000..8b76f93 --- /dev/null +++ b/cdx_toolkit/cli/index.py @@ -0,0 +1,269 @@ +from argparse import Namespace, RawDescriptionHelpFormatter +import logging +import posixpath + +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index +from cdx_toolkit.indexes.resolve import INDEX_BACKENDS +from cdx_toolkit.record_io import ( + FORMAT_CHOICES, canonical_format, format_from_path, import_pyarrow, open_text, overlaps, write_records, +) + +from .common import add_verbose_arg, parse_fields, positive_int + + +LOGGER = logging.getLogger(__name__) + +DESCRIPTION = '''\ +Select captures from an index and write them as JSONL, CDXJ, CSV, TSV or Parquet. + +INDEX is one of: + cc, ia Common Crawl or Internet Archive CDX API + http(s)://... URL of a CDX API server + cc-url-index-s3 Common Crawl's columnar URL index on S3, queried with DuckDB + cc-url-index-hf Common Crawl's columnar URL index on Hugging Face, queried with DuckDB + PATH other columnar (parquet) index: *.parquet files, or a hive-partitioned + directory with --index-backend duckdb + athena://... the columnar URL index queried with AWS Athena, e.g. + athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://bucket/path/ + PATH CDX file(s), local or remote, may be a glob, e.g. + 's3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz' +''' + +EXAMPLES = '''\ +examples: + cdxt index cc --url 'commoncrawl.org/*' --limit 10 + cdxt index cc --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.csv + cdxt index ia --url 'commoncrawl.org/*' --from 2008 --to 200906302359 -o captures.jsonl + cdxt index cc --url 'commoncrawl.org/*' --count + cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.jsonl + cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o captures.csv # in the cdx_toolkit repo + cdxt index 'indexes/cdx-*.gz' --url-file urls.txt -o filtered/ # one output per CDX file + cdxt index cc --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --prefix COMMONCRAWL +''' + + +def add_parser(subparsers): + parser = subparsers.add_parser( + 'index', + help='select captures from an index', + description=DESCRIPTION, + epilog=EXAMPLES, + formatter_class=RawDescriptionHelpFormatter, + ) + add_verbose_arg(parser, subcommand=True) + parser.add_argument('index', metavar='INDEX', help='index name or location (see above)') + parser.add_argument( + '--index-backend', + choices=INDEX_BACKENDS, + default='auto', + help='how INDEX is queried (default: detected from INDEX, see above)', + ) + + selection = parser.add_argument_group( + 'capture selection', + ('--url, --hostname, --domain and --url-file can be repeated and combined; a capture matches if it matches ' + 'any of them.'), + ) + selection.add_argument( + '--url', action='append', dest='urls', default=[], metavar='PATTERN', + help="URL pattern, e.g. 'example.com/*' (prefix), '*.example.com' (domain) or an exact URL", + ) + selection.add_argument( + '--hostname', action='append', dest='hostnames', default=[], metavar='HOST', + help='all captures of this exact hostname, e.g. www.example.com', + ) + selection.add_argument( + '--domain', action='append', dest='domains', default=[], metavar='DOMAIN', + help='all captures of this domain and its subdomains, e.g. example.com', + ) + selection.add_argument( + '--url-file', metavar='PATH', + help=('file with URL patterns (like --url): one per line, or a column of a CSV, TSV, JSONL or Parquet file; ' + 'CDX files only'), + ) + selection.add_argument( + '--url-column', metavar='NAME', help='column of the --url-file with the URL patterns (default: url)' + ) + selection.add_argument( + '--url-file-type', choices=('pattern', 'domain'), + help=('how the --url-file entries match: pattern (default) like --url, e.g. example.com is only that URL; ' + 'domain: example.com matches the domain and its subdomains, example.com/path all URLs starting with it'), + ) + selection.add_argument( + '--crawl', + help=('Common Crawl crawl names (comma separated, e.g. CC-MAIN-2024-33 or CC-MAIN-2018) ' + 'or an integer for the most recent N crawls'), + ) + selection.add_argument('--from', dest='from_ts', metavar='TIMESTAMP', help='start time, e.g. 2008 or 20080328') + selection.add_argument('--to', metavar='TIMESTAMP', help='end time, e.g. 2009 or 200906302359') + selection.add_argument('--closest', metavar='TIMESTAMP', help='captures closest to this timestamp (CDX API only)') + selection.add_argument( + '--filter', action='append', dest='filters', default=[], metavar='[!~=]FIELD:EXPR', + help='CDX API filter, e.g. =status:200 (repeatable; see README for the syntax)', + ) + selection.add_argument( + '--language', action='append', dest='languages', default=[], metavar='CODE', + help=('only captures whose primary content language is CODE (ISO 639-3, e.g. eng or deu; repeatable); ' + 'SQL indexes only'), + ) + selection.add_argument('--url-fgrep', metavar='TEXT', help='only captures whose URL contains TEXT') + selection.add_argument('--url-fgrepv', metavar='TEXT', help='only captures whose URL does not contain TEXT') + selection.add_argument( + '--limit', type=int, + help='maximum number of captures to read from the index (applied before --url-fgrep/--url-fgrepv)', + ) + selection.add_argument( + '--sql', metavar='PATH', + help=("file with a SQL query for Athena or DuckDB ('-' = stdin), used instead of " + '--hostname/--domain/--crawl/--from/--to/--language/--limit'), + ) + + output = parser.add_argument_group('output') + output.add_argument( + '-o', '--output', default='-', metavar='PATH', + help=("output file, local or remote (e.g. s3://...), gzipped if it ends with .gz (default: '-' = stdout); " + 'for CDX files, a path without file extension (or ending with /) is a new or empty directory with one ' + 'output per CDX file'), + ) + output.add_argument( + '--output-format', choices=FORMAT_CHOICES, + help=('output format (default: detected from the --output extension, jsonl for stdout); cdx is the same as ' + "cdxj: Common Crawl's index files have the .cdx extension but use the CDXJ format (crawl index JSON); " + 'parquet needs cdx_toolkit[parquet]; without --fields, CSV, TSV and Parquet output of the CDX API and ' + 'CDX files is written once all captures are read (to include all fields)'), + ) + output.add_argument( + '--fields', type=parse_fields, metavar='FIELD,...', + help='comma-separated fields to output, e.g. url,timestamp,status (default: all fields)', + ) + output.add_argument( + '--count', action='store_true', + help=('only print the number of captures, at most --limit (CDX API: cheap estimate based on the number of ' + 'index pages, overlapping selectors are counted more than once; CDX files and SQL: exact count)'), + ) + output.add_argument( + '--dry-run', action='store_true', + help='only print what would be queried (e.g. the SQL query), without querying the index', + ) + + cdx_files = parser.add_argument_group('CDX files options') + cdx_files.add_argument( + '--processes', type=positive_int, default=1, metavar='N', + help='number of processes, with one output per CDX file (default: 1)', + ) + + sql = parser.add_argument_group('SQL options (Athena, DuckDB)') + sql.add_argument( + '--yes', action='store_true', + help=('do not ask for confirmation before scanning more than 10 crawls ' + '(Athena bills per TB scanned; restrict with --crawl)'), + ) + + cdx_api = parser.add_argument_group('CDX API options') + cdx_api.add_argument('--cc-mirror', metavar='URL', help='use this Common Crawl index mirror') + cdx_api.add_argument('--cc-sort', choices=('mixed', 'ascending'), help='order of the crawls (default: mixed)') + + parser.set_defaults(func=run) + return parser + + +def read_sql(path: str) -> str: + with open_text(path, 'r') as fh: + return fh.read() + + +def build_query(args: Namespace) -> IndexQuery: + return IndexQuery( + urls=args.urls, + hostnames=args.hostnames, + domains=args.domains, + url_file=args.url_file, + url_column=args.url_column, + url_file_type=args.url_file_type, + crawl=args.crawl, + from_ts=args.from_ts, + to=args.to, + closest=args.closest, + filters=args.filters, + url_fgrep=args.url_fgrep, + url_fgrepv=args.url_fgrepv, + limit=args.limit, + languages=args.languages, + sql=read_sql(args.sql) if args.sql else None, + ) + + +def get_output_format(args: Namespace) -> str: + fmt = canonical_format(args.output_format) + if not fmt: + fmt = 'jsonl' if args.output == '-' else format_from_path(args.output) + if fmt is None: + raise UsageError(f'cannot detect the output format of {args.output}, please set --output-format') + if fmt == 'parquet': + if args.output == '-': + raise UsageError('Parquet output cannot be written to stdout, please set --output') + try: + import_pyarrow() + except RuntimeError as e: + raise UsageError(str(e)) from None + return fmt + + +def is_directory_path(path: str) -> bool: + """A path ending with a slash, or whose last component has no file extension (e.g. out/, out, s3://b/out).""" + if path == '-': + return False + path = path.replace('\\', '/') + return path.endswith('/') or not posixpath.splitext(path.rsplit('/', 1)[-1])[1] + + +def writes_per_file(args: Namespace, backend) -> bool: + """CDX files with an output path without file extension: one output per CDX file.""" + from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend + + return ( + isinstance(backend, CdxFilesBackend) + and not (args.count or args.dry_run or args.output_format) + and is_directory_path(args.output) + ) + + +def check_output_is_not_input(output: str, backend, query: IndexQuery) -> None: + """Writing the output must not overwrite what the query reads (the inputs are read while it is written).""" + for location in backend.input_locations(query): + if overlaps(output, location): + raise UsageError(f'the output {output} would overwrite the input {location}; use another --output') + + +def run(args: Namespace, cmdline: str): + backend = open_index( + args.index, args.index_backend, cc_mirror=args.cc_mirror, cc_sort=args.cc_sort, assume_yes=args.yes + ) + query = build_query(args) + check_output_is_not_input(args.output, backend, query) + + if writes_per_file(args, backend): + if args.fields: + raise UsageError('--fields cannot be used when writing one output per CDX file (the lines are copied)') + n_files, n = backend.write_per_file(query, args.output, processes=args.processes) + LOGGER.info('wrote %d captures to %d files in %s', n, n_files, args.output) + return + if args.processes > 1: + raise UsageError('--processes can only be used with CDX files and one output per file (-o DIRECTORY)') + + output_format = None if args.count or args.dry_run else get_output_format(args) + + if args.dry_run: + with open_text(args.output, 'w') as fh: + fh.write(backend.explain(query)) + return + + if args.count: + count = backend.count(query) + with open_text(args.output, 'w') as fh: + fh.write(f'{count}\n') + return + + records = backend.records(query) + n = write_records(records, args.output, output_format, args.fields, fixed_schema=backend.fixed_schema) + LOGGER.info('wrote %d captures to %s', n, args.output) diff --git a/cdx_toolkit/cli/legacy.py b/cdx_toolkit/cli/legacy.py new file mode 100644 index 0000000..8596236 --- /dev/null +++ b/cdx_toolkit/cli/legacy.py @@ -0,0 +1,349 @@ +''' +The command lines of cdx_toolkit 0.9.39 and earlier, e.g. `cdxt --cc --limit 10 iter 'example.com/*'`: options before +the command, and the commands `iter`, `size` and `warc`. They are deprecated: they are parsed with the old grammar and +run with the current code (CDX API backend, repackage()), printing a suggested new command line on stderr. + +Differences to 0.9.39: `warc` writes [-]-001.warc.gz (was -000000.extracted.warc.gz), copies the +WARC records byte-for-byte, writes no file without captures, and exits with status 1 if a capture could not be +fetched; `iter --all-fields --csv` is a usage error (it failed with a traceback). +''' +from argparse import ArgumentParser, Namespace +import csv +import json +import logging +import re +import shlex +import sys +from typing import Iterable, List, Mapping + +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index +from cdx_toolkit.indexes.base import filter_by_url +from cdx_toolkit.utils import get_version + +from .common import IncompleteError + + +LOGGER = logging.getLogger(__name__) + +LEGACY_COMMANDS = ('iter', 'size', 'warc') + +DOCS_URL = 'https://github.com/commoncrawl/cdx_toolkit#command-line-tools' + +# options of the old command line (before the command) that take a value +GLOBAL_VALUE_OPTIONS = { + '--crawl', '--source', '--wb', '--limit', '--cc-mirror', '--cc-sort', '--from', '--to', '--filter', '--closest', +} + +DEFAULT_FIELDS = 'url,status,timestamp' + + +def is_new_top_level_option(token: str) -> bool: + """The options of the new command line before the command: -h, -V and -v.""" + return token in ('-h', '--help', '-V', '--version', '--verbose') or re.fullmatch(r'-v+', token) is not None + + +def is_legacy_command_line(args: List[str]) -> bool: + """A command line of cdx_toolkit 0.9.39: the command is iter, size or warc, or options other than -h, -V and -v + come before the command (the new command line has all other options after the command). The values of the old + options are skipped, so `cdxt --source iter ...` is not taken for the command `iter`.""" + before_command = [] + command = None + tokens = iter(args) + for token in tokens: + if token == '--': + command = next(tokens, None) + break + if token.startswith('-') and token != '-': + before_command.append(token) + if '=' not in token and token in GLOBAL_VALUE_OPTIONS: + next(tokens, None) + continue + command = token + break + return command in LEGACY_COMMANDS or not all(is_new_top_level_option(token) for token in before_command) + + +def build_legacy_parser() -> ArgumentParser: + """The command line of cdx_toolkit 0.9.39.""" + parser = ArgumentParser(prog='cdxt', description='cdx_toolkit iterator command line tool (deprecated syntax)') + + parser.add_argument('--version', '-V', action='version', version=get_version()) + parser.add_argument('--verbose', '-v', action='count', help='set logging level to INFO (-v) or DEBUG (-vv)') + parser.add_argument('--cc', action='store_const', const='cc', help='direct the query to the Common Crawl CDX/WARCs') + parser.add_argument( + '--crawl', + action='store', + help='crawl names (comma separated) or an integer for the most recent N crawls. Implies --cc', + ) + parser.add_argument( + '--ia', action='store_const', const='ia', help='direct the query to the Internet Archive CDX/wayback' + ) + parser.add_argument('--source', action='store', help='direct the query to this CDX server') + parser.add_argument('--wb', action='store', help='direct replays for content to this wayback') + parser.add_argument('--limit', type=int, action='store') + parser.add_argument('--cc-mirror', action='store', help='use this Common Crawl index mirror') + parser.add_argument('--cc-sort', action='store', help='default mixed, alternatively: ascending') + parser.add_argument('--from', action='store') + parser.add_argument('--to', action='store') + parser.add_argument('--filter', action='append', help='see CDX API documentation for usage') + parser.add_argument( + '--get', action='store_true', help='use a single get instead of a paged iteration. default limit=1000' + ) + parser.add_argument( + '--closest', action='store', help='get the closest capture to this timestamp. use with --get' + ) + + subparsers = parser.add_subparsers(dest='cmd') + subparsers.required = True + + iterate = subparsers.add_parser('iter', help='iterate printing captures') + iterate.add_argument('--all-fields', action='store_true') + iterate.add_argument( + '--fields', action='store', default=DEFAULT_FIELDS, help='try --all-fields if you need the list' + ) + iterate.add_argument('--jsonl', action='store_true') + iterate.add_argument('--csv', action='store_true') + iterate.add_argument('url') + iterate.set_defaults(func=run_iter) + + warc = subparsers.add_parser('warc', help='iterate over capture content, creating a warc') + warc.add_argument('--prefix', default='TEST', help='prefix for the warc filename') + warc.add_argument('--subprefix', type=str, default=None, help='subprefix for the warc filename, default None') + warc.add_argument('--size', type=int, default=1000000000, help='target for the warc filesize in bytes') + warc.add_argument('--creator', action='store', help='creator of the warc: person, organization, service') + warc.add_argument('--operator', action='store', help='a person, if the creator is an organization') + warc.add_argument('--url-fgrep', action='store', help='this pattern must be present to warc an url') + warc.add_argument( + '--url-fgrepv', action='store', help='this pattern must not be present to warc an url, e.g. /robots.txt' + ) + warc.add_argument( + '--warc-download-prefix', action='store', help='prefix for downloading content, automatically set for CC' + ) + warc.add_argument('url') + warc.set_defaults(func=run_warc) + + size = subparsers.add_parser('size', help='imprecise count of how many results are available') + size.add_argument('--details', action='store_true', help='show details of each subindex') + size.add_argument('url') + size.set_defaults(func=run_size) + + return parser + + +def legacy_source(ns: Namespace) -> str: + """The CDX API of the old options: --crawl implies cc, otherwise --cc, --ia, then --source.""" + source = 'cc' if ns.crawl else ns.cc or ns.ia or ns.source + if source is None: + raise UsageError('must specify --cc, --ia, or a --source') + if source not in ('cc', 'ia') and not source.startswith(('http://', 'https://')): + raise UsageError(f'--source {source}: use the http(s) URL of a CDX API server') + return source + + +def request_kwargs(ns: Namespace) -> dict: + """The parameters of the CDX API requests (CDXFetcher.iter/get/get_size_estimate), as in 0.9.39.""" + kwargs = {} + if ns.limit: + kwargs['limit'] = ns.limit + if getattr(ns, 'from'): + kwargs['from_ts'] = getattr(ns, 'from') + if ns.to: + kwargs['to'] = ns.to + if ns.closest: + kwargs['closest'] = ns.closest + if ns.filter: + kwargs['filter'] = ns.filter + return kwargs + + +def cc_options(ns: Namespace, source: str) -> dict: + """--cc-mirror and --cc-sort; the old command line ignored them for other sources.""" + if source != 'cc': + return {} + return {'cc_mirror': ns.cc_mirror, 'cc_sort': ns.cc_sort} + + +def fetcher(ns: Namespace, source: str): + """A CDXFetcher for the requests that the index backend does not make as 0.9.39 did (--get, --closest, size).""" + import cdx_toolkit + from cdx_toolkit.commoncrawl import normalize_crawl + + kwargs = {'source': source} + for name, value in cc_options(ns, source).items(): + if value: + kwargs[name] = value + if ns.crawl: + kwargs['crawl'] = normalize_crawl([ns.crawl]) + return cdx_toolkit.CDXFetcher(**kwargs) + + +def legacy_captures(ns: Namespace, source: str, use_get: bool) -> Iterable[Mapping]: + """The captures of the old command line: a single (non-paged) request with --get, the paged iteration with the + closest= parameter for --closest without --get, otherwise the CDX API backend (the same requests).""" + if use_get: + return fetcher(ns, source).get(ns.url, **request_kwargs(ns)) + if ns.closest: + return fetcher(ns, source).iter(ns.url, **request_kwargs(ns)) + backend = open_index(source, 'cdx-api', **cc_options(ns, source)) + query = IndexQuery( + urls=[ns.url], + crawl=ns.crawl, + from_ts=getattr(ns, 'from'), + to=ns.to, + filters=list(ns.filter or []), + limit=ns.limit or None, + ) + return backend.records(query) + + +def iter_fields(ns: Namespace) -> List[str]: + """The fields of `iter --fields` (default url,status,timestamp): without duplicates, sorted.""" + return sorted(set(ns.fields.split(','))) + + +def run_iter(ns: Namespace, cmdline: str): + """`cdxt ... iter URL`: print the captures as in 0.9.39, as text (`key value, ...`), JSON lines (--jsonl) or + CSV (--csv), with the --fields (sorted), or all fields with --all-fields.""" + if ns.all_fields and ns.csv: + # 0.9.39 wrote the header of --fields, then failed with a traceback at the first capture + raise UsageError('--all-fields cannot be used with --csv, select the columns with --fields') + source = legacy_source(ns) + fields = iter_fields(ns) + + writer = None + if ns.csv: + writer = csv.DictWriter(sys.stdout, fieldnames=fields) + writer.writeheader() + + for capture in legacy_captures(ns, source, use_get=ns.get): + if ns.all_fields: + selected = dict(capture) + else: + selected = {k: capture[k] for k in fields if k in capture} + if ns.jsonl: + print(json.dumps(selected, sort_keys=True)) + elif writer: + writer.writerow(selected) + else: + print(', '.join(f'{k} {selected[k]}' for k in sorted(selected))) + + +def run_size(ns: Namespace, cmdline: str): + """`cdxt ... size URL`: print the estimated number of captures (CDXFetcher.get_size_estimate(), not capped by + --limit), with --details first the estimate of each index.""" + kwargs = request_kwargs(ns) + if ns.details: + kwargs['details'] = True # get_size_estimate() prints the estimate of each index + print(fetcher(ns, legacy_source(ns)).get_size_estimate(ns.url, **kwargs)) + + +def warc_content_source(ns: Namespace, source: str) -> dict: + """Where `warc` gets the content of the captures from (as CDXFetcher did): {'wb': ...} or + {'warc_download_prefix': ...}.""" + if ns.wb and ns.warc_download_prefix: + raise UsageError('cannot specify both --wb and --warc-download-prefix') + if ns.wb: + if source == 'cc': + raise UsageError('cannot specify --wb for Common Crawl (--cc, --crawl)') + return {'wb': ns.wb} + if ns.warc_download_prefix: + return {'warc_download_prefix': ns.warc_download_prefix} + if source == 'cc': + return {'warc_download_prefix': 'https://data.commoncrawl.org'} + if source == 'ia': + return {'wb': 'ia'} + raise UsageError('warc with --source needs --wb (a wayback machine) or --warc-download-prefix') + + +def run_warc(ns: Namespace, cmdline: str): + """`cdxt ... warc URL`: write the captures to [-]-001.warc.gz, ... with repackage(), streamed + in index order; IncompleteError (exit status 1) if captures could not be fetched.""" + from cdx_toolkit.repackage import repackage + + source = legacy_source(ns) + content = warc_content_source(ns, source) + # 0.9.39 iterated (paged) also with --get; the limit applies before --url-fgrep/--url-fgrepv + captures = filter_by_url(legacy_captures(ns, source, use_get=False), ns.url_fgrep, ns.url_fgrepv) + result = repackage( + captures, + ns.prefix, + subprefix=ns.subprefix, + size=ns.size, + creator=ns.creator, + operator=ns.operator, + description='warc extraction generated with: ' + cmdline, + sort=False, # stream the captures in index order, as 0.9.39 did + **content, + ) + LOGGER.info('wrote %d of %d captures to WARC files (%d failed)', result.written, result.requested, result.failed) + if result.failed: + raise IncompleteError( + f'{result.failed} of {result.requested} captures could not be fetched (see the log), ' + f'{result.written} were written' + ) + + +def suggested_replacement(ns: Namespace) -> str: + """The new command line that does (about) the same; not exact for the text output of iter, --get, --closest + without --get and size --details.""" + source = 'cc' if ns.crawl else ns.cc or ns.ia or ns.source or 'INDEX' + index = ['cdxt', 'index', source, '--url', ns.url] + if ns.crawl: + index += ['--crawl', ns.crawl] + if ns.limit: + index += ['--limit', str(ns.limit)] + elif ns.cmd == 'iter' and ns.get: + index += ['--limit', '1000'] # the default limit of a single get + for option, value in (('--from', getattr(ns, 'from')), ('--to', ns.to), ('--closest', ns.closest)): + if value: + index += [option, value] + for expression in ns.filter or []: + index += ['--filter', expression] + if source == 'cc': + for option, value in (('--cc-mirror', ns.cc_mirror), ('--cc-sort', ns.cc_sort)): + if value: + index += [option, value] + + if ns.cmd == 'iter': + fields = [] if ns.all_fields else ['--fields', ','.join(iter_fields(ns))] + if ns.jsonl: + index += fields + else: # CSV instead of the old text output + index += ['--output-format', 'csv'] + fields + return render(index) + if ns.cmd == 'size': + return render(index + ['--count']) + + for option, value in (('--url-fgrep', ns.url_fgrep), ('--url-fgrepv', ns.url_fgrepv)): + if value: + index += [option, value] + repackage = ['cdxt', 'repackage', '--prefix', ns.prefix] + if ns.subprefix: + repackage += ['--subprefix', ns.subprefix] + if ns.size != 1000000000: + repackage += ['--size', str(ns.size)] + for option, value in (('--creator', ns.creator), ('--operator', ns.operator)): + if value: + repackage += [option, value] + try: + content = warc_content_source(ns, legacy_source(ns)) + except UsageError: + content = {} + if 'wb' in content: + repackage += ['--wb', content['wb']] + elif content.get('warc_download_prefix', 'https://data.commoncrawl.org') != 'https://data.commoncrawl.org': + repackage += ['--warc-download-prefix', content['warc_download_prefix']] + return render(index) + ' | ' + render(repackage + ['--no-sort']) + + +def render(tokens: List[str]) -> str: + """A command line from its tokens, quoted for the shell.""" + return ' '.join(shlex.quote(token) for token in tokens) + + +def deprecation_message(ns: Namespace) -> str: + """The line printed on stderr for an old command line, with the suggested new command line.""" + return ( + 'cdxt: this command line uses the old syntax, which is deprecated and will be removed in a future version; ' + f'suggested replacement: {suggested_replacement(ns)} (see {DOCS_URL})\n' + ) diff --git a/cdx_toolkit/cli/repackage.py b/cdx_toolkit/cli/repackage.py new file mode 100644 index 0000000..01a4688 --- /dev/null +++ b/cdx_toolkit/cli/repackage.py @@ -0,0 +1,139 @@ +from argparse import Namespace, RawDescriptionHelpFormatter +import logging +import time + +from cdx_toolkit.record_io import FORMAT_CHOICES +from cdx_toolkit.repackage import repackage + +from .common import IncompleteError, add_verbose_arg, parse_size, positive_int + + +LOGGER = logging.getLogger(__name__) + +DESCRIPTION = '''\ +Fetch the captures listed in an index file (as written by `cdxt index`) and write them to WARC files. + +Captures with a WARC location (filename, offset, length; e.g. Common Crawl captures) are copied +byte-for-byte from --warc-download-prefix (https://, s3:// or hf://buckets/...) into +[-]-001.warc.gz, -002, ... (local, s3:// or hf://buckets/...). + +Captures without a WARC location (e.g. Internet Archive captures) are replayed from a wayback +machine with --wb and written to the same series of WARC files. + +WARC files with the same prefix and subprefix must not exist yet. +''' + +EXAMPLES = '''\ +examples: + cdxt index cc --url 'commoncrawl.org/*' --limit 10 -o captures.jsonl + cdxt repackage captures.jsonl --prefix COMMONCRAWL --size 100MB + + cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o homepages.csv # in the cdx_toolkit repo + cdxt repackage homepages.csv --prefix s3://my-bucket/HOMEPAGES --warc-download-prefix s3://commoncrawl \\ + --processes 8 --parallel-readers 16 + + cdxt index ia --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --wb ia --prefix IA +''' + + +def add_parser(subparsers): + parser = subparsers.add_parser( + 'repackage', + help='write the captures of an index file to WARC files', + description=DESCRIPTION, + epilog=EXAMPLES, + formatter_class=RawDescriptionHelpFormatter, + ) + add_verbose_arg(parser, subcommand=True) + parser.add_argument( + 'input', nargs='?', default='-', metavar='INDEX_PATH', + help="index file(s) with captures, local or remote, may be a glob (default: '-' = stdin)", + ) + parser.add_argument( + '--input-format', choices=('auto',) + FORMAT_CHOICES, default='auto', + help=('format of the input file (default: parquet for *.parquet, otherwise detected from the first line); ' + "cdx is the same as cdxj: Common Crawl's index files have the .cdx extension but use the CDXJ format " + '(crawl index JSON)'), + ) + + source = parser.add_argument_group('fetching') + source.add_argument( + '--warc-download-prefix', default='https://data.commoncrawl.org', metavar='URL', + help=('prefix for downloading WARC records by filename (default: %(default)s); s3://commoncrawl is ' + 'fastest in AWS us-east-1, hf://buckets/commoncrawl/commoncrawl reads from Hugging Face'), + ) + source.add_argument( + '--wb', metavar='URL', + help="fetch captures by replaying them from this wayback machine ('ia' = Internet Archive)", + ) + source.add_argument( + '--processes', type=positive_int, default=1, + help='number of worker processes, e.g. the number of vCPUs (default: 1)', + ) + source.add_argument( + '--parallel-readers', type=positive_int, default=1, metavar='N', + help='number of concurrent WARC readers per process (default: 1)', + ) + + output = parser.add_argument_group('output') + output.add_argument( + '--prefix', default='REPACKAGE', + help='prefix (path) of the WARC files, local, s3:// or hf://, e.g. out/CC-HOMEPAGES (default: REPACKAGE)' + ) + output.add_argument('--subprefix', help='subprefix for the WARC filenames') + output.add_argument( + '--size', type=parse_size, default=1000000000, + help='target size of each WARC file, e.g. 1000000000, 100MB or 1GiB (default: 1GB)', + ) + output.add_argument('--creator', help='creator of the WARC: person, organization, service') + output.add_argument('--operator', help='a person, if the creator is an organization') + output.add_argument('--is-part-of', help='isPartOf of the warcinfo record (default: prefix-subprefix)') + output.add_argument('--description', help='description of the warcinfo record (default: the command line)') + output.add_argument( + '--add-metadata-record', action='append', default=[], metavar='PATH', + help='add the content of this file (local or remote) as a metadata record to each WARC (repeatable)', + ) + + advanced = parser.add_argument_group('advanced') + advanced.add_argument( + '--no-sort', action='store_true', + help=('keep the input order instead of sorting by WARC file and offset (sorting improves read ' + 'locality but keeps all captures in memory)'), + ) + advanced.add_argument( + '--log-every-n', type=int, default=1000, metavar='N', + help='log progress every N records (0 = off, default: 1000)', + ) + + parser.set_defaults(func=run) + return parser + + +def run(args: Namespace, cmdline: str): + start_time = time.time() + result = repackage( + args.input, + args.prefix, + subprefix=args.subprefix, + size=args.size, + warc_download_prefix=args.warc_download_prefix, + wb=args.wb, + processes=args.processes, + parallel_readers=args.parallel_readers, + metadata_records=args.add_metadata_record, + sort=not args.no_sort, + log_every_n=args.log_every_n, + creator=args.creator, + operator=args.operator, + is_part_of=args.is_part_of, + description=args.description or 'warc extraction generated with: ' + cmdline, + input_format=args.input_format, + ) + + LOGGER.info('wrote %d of %d captures to WARC files (%d failed) in %.1f seconds', + result.written, result.requested, result.failed, time.time() - start_time) + if result.failed: + raise IncompleteError( + f'{result.failed} of {result.requested} captures could not be fetched (see the log), ' + f'{result.written} were written' + ) diff --git a/cdx_toolkit/hf.py b/cdx_toolkit/hf.py new file mode 100644 index 0000000..049df76 --- /dev/null +++ b/cdx_toolkit/hf.py @@ -0,0 +1,33 @@ +''' +Hugging Face Storage Buckets (hf://buckets/...): the version of huggingface_hub they need. +''' +import re +from typing import Tuple + +# HfFileSystem lists and opens bucket paths since 1.6.0; before 1.7.2, listing a missing path raised +# EntryNotFoundError instead of FileNotFoundError (which the checks of existing outputs rely on) +HF_HUB_MIN_VERSION = (1, 7, 2) + + +def version_tuple(version: str) -> Tuple[int, ...]: + """'1.7.2' -> (1, 7, 2), '2.0.0rc1' -> (2, 0, 0)""" + return tuple(int(re.match(r'\d*', part).group() or 0) for part in version.split('.')[:3]) + + +def check_huggingface_hub() -> None: + """Raise RuntimeError with a clear message if huggingface_hub is missing or too old for hf://buckets/ paths + (an installed older version also satisfies a requirement without a minimum).""" + minimum = '.'.join(map(str, HF_HUB_MIN_VERSION)) + install = f"pip install 'cdx_toolkit[hf]' (or 'huggingface_hub>={minimum}')" + try: + import huggingface_hub + except ImportError: + raise RuntimeError(f'hf://buckets/ paths need huggingface_hub: {install}') from None + version = huggingface_hub.__version__ + if version_tuple(version) < HF_HUB_MIN_VERSION: + raise RuntimeError(f'hf://buckets/ paths need huggingface_hub >= {minimum}, not {version}: {install}') + + +def uses_hf(args) -> bool: + """Whether command line arguments name a Hugging Face bucket.""" + return any('hf://' in arg or arg == 'cc-url-index-hf' for arg in args) diff --git a/cdx_toolkit/indexes/__init__.py b/cdx_toolkit/indexes/__init__.py new file mode 100644 index 0000000..9040ef7 --- /dev/null +++ b/cdx_toolkit/indexes/__init__.py @@ -0,0 +1,3 @@ +from .base import IndexBackend, QueryPlan, UsageError # noqa: F401 +from .query import IndexQuery # noqa: F401 +from .resolve import open_index # noqa: F401 diff --git a/cdx_toolkit/indexes/backends/__init__.py b/cdx_toolkit/indexes/backends/__init__.py new file mode 100644 index 0000000..3835f43 --- /dev/null +++ b/cdx_toolkit/indexes/backends/__init__.py @@ -0,0 +1,4 @@ +''' +The index backends, one module each: the CDX API (cdx_api), CDX files (cdx_files), and the columnar URL index +queried with SQL (sql: common logic, sql_builder: the SQL text) via DuckDB (duckdb) or AWS Athena (athena). +''' diff --git a/cdx_toolkit/indexes/backends/athena.py b/cdx_toolkit/indexes/backends/athena.py new file mode 100644 index 0000000..2dc34de --- /dev/null +++ b/cdx_toolkit/indexes/backends/athena.py @@ -0,0 +1,194 @@ +''' +Index backend for the columnar URL index queried with AWS Athena (boto3). +''' +from dataclasses import dataclass +import logging +import re +import time +from typing import Iterator, Optional +from urllib.parse import parse_qs, urlparse + +from cdx_toolkit.aws import botocore_config + +from ..base import UsageError +from .sql import SqlBackend + + +logger = logging.getLogger(__name__) + +# Athena pricing is ~$5 per TB scanned (used for the post-run cost estimate). +ATHENA_USD_PER_TB = 5.0 + + +class AthenaBackend(SqlBackend): + """The columnar index queried with AWS Athena. + + Location: athena://athena..amazonaws.com[:443]/?S3OutputLocation=s3://...[&table=ccindex] + """ + + name = 'Athena' + engine = 'athena' + url_file_hint = ( + ' (after creating an Athena table `target_urls` from the URL file on S3):\n\n' + ' SELECT idx.url, idx.warc_filename, idx.warc_record_offset, idx.warc_record_length\n' + ' FROM ccindex AS idx\n' + ' JOIN target_urls AS target ON idx.url = target.url\n' + " WHERE idx.crawl = 'CC-MAIN-2024-33' AND idx.subset = 'warc'" + ) + + def __init__(self, location: str, assume_yes: bool = False): + super().__init__(location, assume_yes) + self.athena = AthenaLocation.parse(location) + + def from_clause(self, crawls): + return self.athena.table, False + + def iter_sql_rows(self, sql): + client = AthenaClient(self.athena.database, self.athena.s3_output_location, region_name=self.athena.region) + return client.iter_rows(sql) + + +@dataclass(frozen=True) +class AthenaLocation: + """Where an index is queried with Athena, and where Athena writes the results.""" + + region: str + database: str + table: str + s3_output_location: str + + @classmethod + def parse(cls, location: str) -> 'AthenaLocation': + """athena://athena.us-east-1.amazonaws.com:443/ccindex?S3OutputLocation=s3://bucket/path/[&table=ccindex]""" + parsed = urlparse(location) + match = re.fullmatch(r'athena\.([a-z0-9-]+)\.amazonaws\.com', parsed.hostname or '') + if parsed.scheme != 'athena' or not match: + raise UsageError( + f'invalid Athena location {location!r}, expected ' + 'athena://athena..amazonaws.com/?S3OutputLocation=s3:////' + ) + params = parse_qs(parsed.query) + s3_output_location = params.get('S3OutputLocation', [None])[0] + if not s3_output_location or not s3_output_location.startswith('s3://'): + raise UsageError(f'the Athena location {location!r} needs a S3OutputLocation=s3://... parameter') + database = parsed.path.strip('/') or 'ccindex' + table = params.get('table', ['ccindex'])[0] + return cls(match.group(1), database, table, s3_output_location) + + +def make_client(region_name: Optional[str] = None): + import boto3 + + # retries: Common Crawl's recommended AWS settings, unless configured by the user (see cdx_toolkit.aws) + return boto3.client('athena', config=botocore_config(region_name=region_name, read_timeout=60)) + + +class AthenaClient: + """Runs queries in an Athena database; Athena writes the results to `s3_output_location`.""" + + def __init__(self, database: str, s3_output_location: str, region_name: Optional[str] = None, client=None, + max_wait_time: int = 300): + self.database = database + self.s3_output_location = s3_output_location + self.client = client or make_client(region_name) + self.max_wait_time = max_wait_time + + def iter_rows(self, sql: str) -> Iterator[dict]: + """Run the query and yield the result rows as dicts (column name -> string value).""" + query_execution_id = self.run(sql) + try: + yield from self.rows(query_execution_id) + finally: + self.report_cost(query_execution_id) + + def run(self, sql: str) -> str: + """Start an Athena query and block until it completes; return the execution id. + + Raises if the query does not reach the SUCCEEDED state. If the wait is + interrupted (Ctrl-C / cancellation) or times out, the query is cancelled + server-side so we don't keep paying for a scan whose results we'll never read.""" + logger.info('Executing Athena query: %s', sql) + + response = self.client.start_query_execution( + QueryString=sql, + QueryExecutionContext={'Database': self.database}, + ResultConfiguration={'OutputLocation': self.s3_output_location}, + ) + + query_execution_id = response['QueryExecutionId'] + logger.info('Query execution started. ID: %s', query_execution_id) + + try: + status = self._wait_for_completion(query_execution_id) + except BaseException: + # Ctrl-C, asyncio cancellation, or timeout: stop the query server-side + # to bound the Athena scan cost, then propagate the original exception. + self._stop(query_execution_id) + raise + + if status != 'SUCCEEDED': + raise Exception(f'Query failed with status: {status}') + + return query_execution_id + + def rows(self, query_execution_id: str) -> Iterator[dict]: + """The results of a completed query as dicts (column name -> string value).""" + paginator = self.client.get_paginator('get_query_results') + column_names = None + + for page in paginator.paginate(QueryExecutionId=query_execution_id): + rows = page['ResultSet']['Rows'] + + # Get column names from first page + if column_names is None and rows: + column_names = [col['VarCharValue'] for col in rows[0]['Data']] + rows = rows[1:] # Skip header row + + for row in rows: + row_data = [cell.get('VarCharValue', None) for cell in row['Data']] + yield dict(zip(column_names, row_data)) + + def report_cost(self, query_execution_id: str) -> None: + """Log the bytes scanned and an estimated USD cost for a completed query.""" + try: + response = self.client.get_query_execution(QueryExecutionId=query_execution_id) + scanned = response['QueryExecution'].get('Statistics', {}).get('DataScannedInBytes') + except Exception as e: # pragma: no cover - best-effort reporting + logger.debug('unable to read Athena query statistics: %r', e) + return + + if scanned is None: + return + + gb = scanned / 1e9 + usd = scanned / 1e12 * ATHENA_USD_PER_TB + logger.info('Athena scanned %.2f GB, estimated cost ~$%.4f', gb, usd) + + def _wait_for_completion(self, query_execution_id: str) -> str: + """Wait for query to complete and return final status""" + start_time = time.time() + + while time.time() - start_time < self.max_wait_time: + response = self.client.get_query_execution(QueryExecutionId=query_execution_id) + + status = response['QueryExecution']['Status']['State'] + logger.info(f'Query status: {status}') + + if status in ['SUCCEEDED', 'FAILED', 'CANCELLED']: + if status == 'FAILED': + error_reason = response['QueryExecution']['Status'].get('StateChangeReason', 'Unknown error') + logger.info(f'Query failed: {error_reason}') + return status + + time.sleep(2) + + raise TimeoutError(f'Query did not complete within {self.max_wait_time} seconds') + + def _stop(self, query_execution_id: str) -> None: + """Best-effort cancellation of a running Athena query.""" + logger.warning('Cancelling Athena query %s ...', query_execution_id) + try: + self.client.stop_query_execution(QueryExecutionId=query_execution_id) + logger.warning('Athena query %s cancelled', query_execution_id) + except Exception as e: # pragma: no cover - best-effort cleanup + logger.warning('Failed to cancel Athena query %s: %r', query_execution_id, e) diff --git a/cdx_toolkit/indexes/backends/cdx_api.py b/cdx_toolkit/indexes/backends/cdx_api.py new file mode 100644 index 0000000..9f66bed --- /dev/null +++ b/cdx_toolkit/indexes/backends/cdx_api.py @@ -0,0 +1,158 @@ +from dataclasses import dataclass +import logging +from typing import Iterator, List, Mapping, Optional + +import cdx_toolkit +from cdx_toolkit.commoncrawl import normalize_crawl +from cdx_toolkit.compat import munge_filter + +from ..base import IndexBackend, QueryPlan, UsageError +from ..patterns import normalize_host, url_hostname +from ..query import IndexQuery + + +LOGGER = logging.getLogger(__name__) + + +class CdxApiBackend(IndexBackend): + """A CDX API server: Common Crawl ('cc'), the Internet Archive ('ia'), or any pywb/wayback URL.""" + + name = 'CDX API' + supported_options = frozenset({ + 'urls', 'hostnames', 'domains', 'crawl', 'from_ts', 'to', 'closest', 'filters', + 'url_fgrep', 'url_fgrepv', 'limit', + }) + + def __init__(self, source: str, cc_mirror: Optional[str] = None, cc_sort: Optional[str] = None): + self.source = source + self.cc_mirror = cc_mirror + self.cc_sort = cc_sort + + def validate(self, query: IndexQuery) -> None: + super().validate(query) + if not query.url_patterns(): + raise UsageError('the CDX API needs at least one --url, --hostname or --domain') + if query.crawl and self.source != 'cc': + raise UsageError('--crawl can only be used with the Common Crawl index (cc)') + for expression in query.filters: + # the check of CDXFetcher, which would only run when the captures are read + try: + munge_filter([expression], self.source) + except ValueError as e: + raise UsageError( + f'--filter {expression!r}: {e} (the Internet Archive supports only FIELD:REGEX and ' + '!FIELD:REGEX, e.g. status:200)' + ) from None + + def plan(self, query: IndexQuery) -> 'ApiPlan': + requests = [] + for url, match_type in query.url_patterns(): + hostname = normalize_host(url) if match_type == 'host' else None + requests.append(ApiRequest(hostname or url, api_params(query, match_type), hostname=hostname)) + return ApiPlan(self.source, requests, query.limit, query.crawl) + + def select(self, query: IndexQuery) -> Iterator[Mapping]: + """The captures of all selectors; a capture selected by more than one is only returned once.""" + plan = self.plan(query) + cdx = self._fetcher(query) + seen = set() if len(plan.requests) > 1 else None + for request in plan.requests: + params = request.params_with(plan.server_limit) + # closest only makes sense for a single (non-paged) query + captures = cdx.get(request.url, **params) if query.closest else cdx.iter(request.url, **params) + for capture in captures: + if request.hostname and url_hostname(capture.get('url', '')) != request.hostname: + continue + if seen is not None: + identity = capture_identity(capture) + if identity in seen: + continue + seen.add(identity) + yield capture + + def select_count(self, query: IndexQuery) -> int: + """Estimate from the number of index pages, at most --limit; overlapping selectors are counted more + than once, and --hostname counts the host with and without www. (the server's matchType=host).""" + cdx = self._fetcher(query) + count = sum( + cdx.get_size_estimate(request.url, **request.params_with(query.limit)) + for request in self.plan(query).requests + ) + return min(count, query.limit) if query.limit else count + + def _fetcher(self, query: IndexQuery) -> cdx_toolkit.CDXFetcher: + kwargs = {'source': self.source} + if self.cc_mirror: + kwargs['cc_mirror'] = self.cc_mirror + if self.cc_sort: + kwargs['cc_sort'] = self.cc_sort + if query.crawl: + kwargs['crawl'] = normalize_crawl([query.crawl]) + return cdx_toolkit.CDXFetcher(**kwargs) + + +@dataclass +class ApiRequest: + """One request (series of paged requests) to the CDX API: a URL and its parameters (without the limit). + + matchType=host selects the SURT host, which is the same for example.com and www.example.com: with `hostname`, + only the captures of exactly this host are kept.""" + + url: str + params: dict + hostname: Optional[str] = None + + def params_with(self, limit: Optional[int]) -> dict: + return {'limit': limit, **self.params} if limit else dict(self.params) + + +@dataclass +class ApiPlan(QueryPlan): + """One request per URL selector (--url, --hostname, --domain).""" + + source: str + requests: List[ApiRequest] + limit: Optional[int] = None + crawl: Optional[str] = None + + @property + def server_limit(self) -> Optional[int]: + """The limit sent to the server: only for a single request whose captures are all kept; with several, the + captures selected by more than one request are removed first, and with --hostname, the captures of the + other host (with or without www.), and the limit is applied by IndexBackend.records().""" + return self.limit if len(self.requests) == 1 and not self.requests[0].hostname else None + + def describe(self) -> str: + lines = [] + for request in self.requests: + params = ' '.join(f'{k}={v}' for k, v in request.params_with(self.server_limit).items()) + lines.append(f'{self.source}: url={request.url} {params}'.rstrip()) + if request.hostname: + lines.append(f' only captures of the host {request.hostname}') + if self.limit and self.server_limit is None: + lines.append(f'limit: {self.limit} (after removing duplicates and other hosts)') + if self.crawl: + lines.append(f'crawls: {self.crawl}') + return '\n'.join(lines) + '\n' + + +def api_params(query: IndexQuery, match_type: Optional[str]) -> dict: + """The parameters of a CDX API request (CDXFetcher.iter/get) for one URL selector.""" + params = {} + if query.from_ts: + params['from_ts'] = query.from_ts + if query.to: + params['to'] = query.to + if query.closest: + params['closest'] = query.closest + if query.filters: + params['filter'] = list(query.filters) + if match_type: + params['matchType'] = match_type + return params + + +def capture_identity(capture: Mapping) -> tuple: + """What distinguishes captures: the same URL can be captured at the same time more than once (e.g. in + different crawls), so the WARC location is part of the identity.""" + return tuple(capture.get(field) for field in ('urlkey', 'url', 'timestamp', 'filename', 'offset', 'digest')) diff --git a/cdx_toolkit/indexes/backends/cdx_files.py b/cdx_toolkit/indexes/backends/cdx_files.py new file mode 100644 index 0000000..432f1dc --- /dev/null +++ b/cdx_toolkit/indexes/backends/cdx_files.py @@ -0,0 +1,295 @@ +''' +Index backend for CDX files (e.g. Common Crawl's cc-index/collections/*/indexes/cdx-*.gz), local or remote. + +Each line is ` ` (CDXJ). Captures are selected by matching the SURT key +against SURT prefixes (with url-is-in) or exact SURT keys, built from --url, --hostname, --domain +and --url-file. +''' +import logging +import os +from dataclasses import dataclass +from itertools import islice +from typing import Iterator, List, Mapping, NamedTuple, Optional, Tuple + +from cdx_toolkit.aws import process_pool +from cdx_toolkit.record_io import ( + check_glob, format_from_path, is_empty_dir, is_glob, open_text, parse_cdxj_line, read_records, +) +from cdx_toolkit.timeutils import pad_timestamp, pad_timestamp_up + +from ..base import IndexBackend, QueryPlan, UsageError +from ..patterns import ( + domain_entry_to_surts, domain_to_surts, hostname_to_surts, normalize_host, url_hostname, url_pattern_to_surts, +) +from ..query import IndexQuery + + +LOGGER = logging.getLogger(__name__) + +GLOB_CHARS = ('*', '?', '[') + +# file formats of --url-file that have columns (others: one URL pattern per line) +TABLE_FORMATS = ('csv', 'tsv', 'jsonl', 'parquet') + + +URL_FILE_TYPES = ('pattern', 'domain') + + +def load_url_patterns(path: str, column: Optional[str] = None) -> List[str]: + """URL patterns from a file: one per line (e.g. .txt), or a column of a CSV, TSV, JSONL or Parquet file.""" + fmt = format_from_path(path) + if fmt in TABLE_FORMATS: + column = column or 'url' + patterns = [] + for i, record in enumerate(read_records(path, fmt)): + if column not in record: + if i == 0: + raise UsageError(f'--url-file {path} has no column {column!r} (use --url-column)') + continue + if record[column]: + patterns.append(str(record[column])) + return patterns + if column: + raise UsageError(f'--url-column needs a CSV, TSV, JSONL or Parquet --url-file, not {path}') + with open_text(path, 'r') as fh: + return [line.strip() for line in fh if line.strip() and not line.startswith('#')] + + +@dataclass(frozen=True) +class CaptureFilter: + """Which lines of a CDX file are selected (picklable, for worker processes).""" + + prefixes: Tuple[str, ...] = () + exact: Tuple[str, ...] = () + # --hostname: the SURT prefixes select the candidates (with and without www.), the URL's host must be one of + # the hostnames + host_prefixes: Tuple[str, ...] = () + hostnames: Tuple[str, ...] = () + select_all: bool = False + from_ts: Optional[str] = None + to: Optional[str] = None + url_fgrep: Optional[str] = None + url_fgrepv: Optional[str] = None + + @classmethod + def from_query(cls, query: IndexQuery, url_filters: bool = False) -> 'CaptureFilter': + """The selection of the query; --url-fgrep/--url-fgrepv only with `url_filters` (one output per file: + without --limit, they can be applied while reading; otherwise they are applied after the limit).""" + prefixes, exact, host_prefixes = [], [], [] + patterns = list(query.urls) + if query.url_file: + entries = load_url_patterns(query.url_file, query.url_column) + LOGGER.info('loaded %d entries from %s', len(entries), query.url_file) + if (query.url_file_type or 'pattern') == 'domain': + for entry in entries: + prefixes += domain_entry_to_surts(entry) + else: + patterns += entries + for pattern in patterns: + p, e = url_pattern_to_surts(pattern) + prefixes += p + exact += e + for hostname in query.hostnames: + host_prefixes += hostname_to_surts(hostname) + for domain in query.domains: + prefixes += domain_to_surts(domain) + return cls( + prefixes=tuple(prefixes), + exact=tuple(exact), + host_prefixes=tuple(host_prefixes), + hostnames=tuple(normalize_host(hostname) for hostname in query.hostnames), + select_all=not (query.urls or query.url_file or query.hostnames or query.domains), + from_ts=pad_timestamp(query.from_ts) if query.from_ts else None, + to=pad_timestamp_up(query.to) if query.to else None, + url_fgrep=query.url_fgrep if url_filters else None, + url_fgrepv=query.url_fgrepv if url_filters else None, + ) + + def matcher(self) -> 'LineMatcher': + return LineMatcher(self) + + +class LineMatcher: + def __init__(self, capture_filter: CaptureFilter): + from url_is_in import SURTMatcher + + self.filter = capture_filter + self.exact = set(capture_filter.exact) + self.prefixes = None + if capture_filter.prefixes: + self.prefixes = SURTMatcher(list(capture_filter.prefixes), match_subdomains=False) + self.host_prefixes = None + if capture_filter.host_prefixes: + self.host_prefixes = SURTMatcher(list(capture_filter.host_prefixes), match_subdomains=False) + self.hostnames = set(capture_filter.hostnames) + + def selected(self, key: str, line: str) -> bool: + """Selected by --url, --hostname, --domain or --url-file.""" + if self.filter.select_all or key in self.exact or (self.prefixes and self.prefixes.is_in(key)): + return True + if self.host_prefixes and self.host_prefixes.is_in(key): + return url_hostname(parse_cdxj_line(line).get('url', '')) in self.hostnames + return False + + def matches(self, line: str) -> bool: + key, _, rest = line.partition(' ') + f = self.filter + if not self.selected(key, line): + return False + if f.from_ts or f.to: + timestamp = rest[:rest.find(' ')] + if (f.from_ts and timestamp < f.from_ts) or (f.to and timestamp > f.to): + return False + if f.url_fgrep or f.url_fgrepv: + url = parse_cdxj_line(line).get('url', '') + if (f.url_fgrep and f.url_fgrep not in url) or (f.url_fgrepv and f.url_fgrepv in url): + return False + return True + + +def iter_matching_lines(path: str, matcher: LineMatcher) -> Iterator[str]: + n_lines = n_matches = 0 + with open_text(path, 'r') as fh: + for line in fh: + n_lines += 1 + if line.strip() and matcher.matches(line): + n_matches += 1 + yield line + LOGGER.info('%s: %d of %d lines selected', path, n_matches, n_lines) + + +def _filter_file(task: Tuple[str, str, CaptureFilter]) -> Tuple[str, int]: + """Write the selected lines of one CDX file to its own output (worker process); no file without matches.""" + input_path, output_path, capture_filter = task + matcher = capture_filter.matcher() + n = 0 + fh = ctx = None + try: + for line in iter_matching_lines(input_path, matcher): + if fh is None: + if '://' not in output_path: + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + ctx = open_text(output_path, 'w') + fh = ctx.__enter__() + fh.write(line if line.endswith('\n') else line + '\n') + n += 1 + finally: + if ctx is not None: + ctx.__exit__(None, None, None) + return output_path, n + + +class CdxFilesBackend(IndexBackend): + """CDX files (CDXJ, optionally gzipped), local or remote (fsspec), a single file or a glob.""" + + name = 'CDX files' + supported_options = frozenset({ + 'urls', 'hostnames', 'domains', 'url_file', 'url_column', 'url_file_type', 'from_ts', 'to', + 'url_fgrep', 'url_fgrepv', 'limit', + }) + + def __init__(self, location: str): + self.location = location + + def validate(self, query: IndexQuery) -> None: + super().validate(query) + try: + check_glob(self.location) + except ValueError as e: + raise UsageError(str(e)) from None + if (query.url_column or query.url_file_type) and not query.url_file: + raise UsageError('--url-column and --url-file-type need --url-file') + if query.url_file_type and query.url_file_type not in URL_FILE_TYPES: + choices = ', '.join(URL_FILE_TYPES) + raise UsageError(f'unknown --url-file-type {query.url_file_type!r} (choose from {choices})') + if not (query.urls or query.url_file or query.hostnames or query.domains or query.from_ts or query.to): + raise UsageError( + f'select captures from the {self.name} with --url, --hostname, --domain, --url-file, --from or --to' + ) + for pattern in query.urls: + url_pattern_to_surts(pattern) + + def input_locations(self, query: IndexQuery) -> List[str]: + return [self.location] + ([query.url_file] if query.url_file else []) + + def files(self) -> List['CdxFile']: + """The CDX files of the location (a file or a glob).""" + import fsspec + + if not is_glob(self.location): + if '://' in self.location: + return [CdxFile(self.location, self.location.split('?', 1)[0].rstrip('/').rsplit('/', 1)[-1])] + return [CdxFile(self.location, os.path.basename(self.location))] # also with backslashes on Windows + fs, fs_path = fsspec.url_to_fs(self.location) + first_glob = min(fs_path.index(c) for c in GLOB_CHARS if c in fs_path) + base = fs_path[:fs_path.rfind('/', 0, first_glob)] + paths = sorted(fs.glob(fs_path)) + if not paths: + raise FileNotFoundError(f'no files found for {self.location}') + return [CdxFile(fs.unstrip_protocol(p), p[len(base):].lstrip('/')) for p in paths] + + def plan(self, query: IndexQuery) -> 'CdxFilesPlan': + return CdxFilesPlan(self.location, self.files(), CaptureFilter.from_query(query)) + + def select(self, query: IndexQuery) -> Iterator[Mapping]: + plan = self.plan(query) + matcher = plan.capture_filter.matcher() + for cdx_file in plan.files: + for line in iter_matching_lines(cdx_file.path, matcher): + yield parse_cdxj_line(line) + + def select_count(self, query: IndexQuery) -> int: + """Exact number of selected captures, at most --limit (reads all files, or until the limit).""" + return sum(1 for _ in islice(self.select(query), query.limit)) + + def write_per_file(self, query: IndexQuery, output_base: str, processes: int = 1) -> Tuple[int, int]: + """Write the selected lines of each CDX file, unchanged, to /. + + Returns the number of output files and captures; no file is written without matches.""" + self.validate(query) + if query.limit: + raise UsageError('--limit cannot be used when writing one output per CDX file') + if not is_empty_dir(output_base): + raise UsageError( + f'the output directory {output_base} is not empty; use a new directory, so that no output of an ' + 'earlier run is mixed with this one' + ) + capture_filter = CaptureFilter.from_query(query, url_filters=True) + join = (lambda rel: output_base.rstrip('/') + '/' + rel) if '://' in output_base else \ + (lambda rel: os.path.join(output_base, *rel.split('/'))) + tasks = [(f.path, join(f.relative_path), capture_filter) for f in self.files()] + + if processes > 1: + with process_pool(processes) as ex: + results = list(ex.map(_filter_file, tasks)) + else: + results = [_filter_file(task) for task in tasks] + + written = [(path, n) for path, n in results if n] + LOGGER.info('wrote %d captures to %d of %d files', sum(n for _, n in written), len(written), len(tasks)) + return len(written), sum(n for _, n in written) + + +class CdxFile(NamedTuple): + path: str + relative_path: str # to the base directory of the glob (the path of the output with one output per file) + + +@dataclass +class CdxFilesPlan(QueryPlan): + """The CDX files and which of their lines are selected.""" + + location: str + files: List[CdxFile] + capture_filter: CaptureFilter + + def describe(self) -> str: + f = self.capture_filter + lines = [f'{len(self.files)} CDX file(s): {self.location}'] + lines += [f' {cdx_file.path}' for cdx_file in self.files[:20]] + ([' ...'] if len(self.files) > 20 else []) + lines.append(f'{len(f.prefixes)} SURT prefix(es), {len(f.exact)} exact URL(s)') + lines += [f' prefix {p}' for p in f.prefixes[:20]] + lines += [f' exact {e}' for e in f.exact[:20]] + if f.from_ts or f.to: + lines.append(f'timestamps: {f.from_ts or "-"} to {f.to or "-"}') + return '\n'.join(lines) + '\n' diff --git a/cdx_toolkit/indexes/backends/duckdb.py b/cdx_toolkit/indexes/backends/duckdb.py new file mode 100644 index 0000000..19a5b87 --- /dev/null +++ b/cdx_toolkit/indexes/backends/duckdb.py @@ -0,0 +1,128 @@ +''' +Index backend for the columnar URL index queried with DuckDB, e.g. on S3, on Hugging Face or local. +''' +import logging +from typing import Iterator, List, Optional + +from cdx_toolkit.aws import aws_region, duckdb_settings + +from ..base import UsageError +from .sql import SqlBackend + + +logger = logging.getLogger(__name__) + +try: + import duckdb + _HAS_DUCKDB = True +except ImportError: # pragma: no cover - exercised in minimal installs + duckdb = None + _HAS_DUCKDB = False + + +def build_from_clause(index_path: str, crawls: Optional[List[str]]) -> str: + """Build a read_parquet(...) FROM expression over the CC columnar index. + + When crawls are given we glob only those crawl partitions (the pruning lever); + otherwise we glob every crawl (expensive -> the cost guard fires).""" + base = index_path.rstrip('/') + if crawls: + globs = [f"'{base}/crawl={c}/subset=warc/*.parquet'" for c in crawls] + else: + globs = [f"'{base}/crawl=*/subset=warc/*.parquet'"] + return f"read_parquet([{', '.join(globs)}], hive_partitioning=true)" + + +def _configure_httpfs(con, s3: bool, region_name: Optional[str] = None) -> None: + """Load the httpfs extension (http(s):// and s3://); for s3://, also find AWS credentials and apply the S3 + settings.""" + con.execute('INSTALL httpfs; LOAD httpfs;') + settings = ['SET http_timeout=120;'] # 2 minutes for large parquet partitions + if s3: + region = aws_region(region_name) + try: + # find AWS credentials like boto3 does (env, ~/.aws profiles, SSO, instance roles) + con.execute('INSTALL aws; LOAD aws;') + con.execute(f"CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION '{region}');") + except Exception as e: # pragma: no cover - no credentials or aws extension unavailable + logger.debug('duckdb AWS credential chain not used: %r', e) + # region, retries and URL style: Common Crawl's recommended S3 settings, unless configured by + # the user (see cdx_toolkit.aws) + settings = duckdb_settings(region_name) + settings + for stmt in settings: + try: + con.execute(stmt) + except Exception as e: # pragma: no cover - setting unsupported on this duckdb + logger.debug('duckdb setting skipped: %s (%r)', stmt, e) + + +def iter_duckdb_rows(query: str, region_name: Optional[str] = None) -> Iterator[dict]: + """Run the query and yield the result rows as dicts (column name -> value). + + AWS region/credentials come from the environment (the AWS credential chain); the + public bucket is readable with valid credentials.""" + if not _HAS_DUCKDB: + raise RuntimeError( + "DuckDB engine requires optional dependencies. Install them with: pip install 'cdx_toolkit[duckdb]'" + ) + + logger.info('Executing DuckDB query: %s', query) + + con = duckdb.connect() + try: + # CC timestamps are UTC (timestamp literals in queries, timestamps in the results) + con.execute("SET TimeZone='UTC';") + # remote files only as needed: a local query needs no extensions (nor AWS credentials) + if 's3://' in query or 'http://' in query or 'https://' in query: + _configure_httpfs(con, s3='s3://' in query, region_name=region_name) + if 'hf://buckets/' in query: + # DuckDB cannot read Hugging Face Storage Buckets itself; use their fsspec filesystem. + from cdx_toolkit.hf import check_huggingface_hub + + check_huggingface_hub() + from huggingface_hub import HfFileSystem + + con.register_filesystem(HfFileSystem()) + cur = con.execute(query) + + col_names = [d[0] for d in cur.description] + + while True: + rows = cur.fetchmany(1000) + if not rows: + break + for row in rows: + yield dict(zip(col_names, row)) + finally: + con.close() + + +class DuckDbBackend(SqlBackend): + """The columnar index queried with DuckDB: a hive-partitioned directory or parquet files (local, s3, hf).""" + + name = 'DuckDB' + engine = 'duckdb' + url_file_hint = ( + ':\n\n' + ' SELECT idx.url, idx.warc_filename, idx.warc_record_offset, idx.warc_record_length\n' + " FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-33/subset=warc/" + "*.parquet') AS idx\n" + " JOIN read_csv('urls.csv') AS target ON idx.url = target.url" + ) + + def __init__(self, location: str, assume_yes: bool = False): + if "'" in location: + raise UsageError(f'invalid index location {location!r}') + super().__init__(location, assume_yes) + + def from_clause(self, crawls): + if self.location.endswith('.parquet'): # file(s), possibly a glob + return f"read_parquet('{self.location}', hive_partitioning=true)", False + # hive-partitioned index directory: prune the crawls in the glob + return build_from_clause(self.location, crawls), True + + def input_locations(self, query): + return [self.location] + + def iter_sql_rows(self, sql): + return iter_duckdb_rows(sql) diff --git a/cdx_toolkit/indexes/backends/sql.py b/cdx_toolkit/indexes/backends/sql.py new file mode 100644 index 0000000..e4372fc --- /dev/null +++ b/cdx_toolkit/indexes/backends/sql.py @@ -0,0 +1,177 @@ +''' +The common logic of the backends for the columnar URL index (parquet), queried with SQL (see duckdb.py, athena.py). +''' +from abc import abstractmethod +from dataclasses import dataclass +import datetime +import logging +import re +import sys +from typing import Iterator, List, Mapping, Optional, Tuple + +from ..base import IndexBackend, QueryPlan, UsageError +from ..query import IndexQuery, field_flag +from .sql_builder import build_sql, query_conditions, resolve_crawl_names, url_surtkeys + + +LOGGER = logging.getLogger(__name__) + +# columns of the columnar index selected by the guided (--hostname/--domain) queries, +# mapped to the field names of the CDX API so records look the same for every backend +COLUMNS = { + 'url_surtkey': 'urlkey', + 'fetch_time': 'timestamp', + 'url': 'url', + 'content_mime_type': 'mime', + 'content_mime_detected': 'mime-detected', + 'fetch_status': 'status', + 'content_digest': 'digest', + 'warc_record_length': 'length', + 'warc_record_offset': 'offset', + 'warc_filename': 'filename', + 'content_languages': 'languages', + 'content_charset': 'encoding', + 'fetch_redirect': 'redirect', + 'crawl': 'crawl', +} + +# A query restricted to at most this many crawls is considered cheap enough to +# run without a cost-confirmation prompt. +LARGE_CRAWL_SET_THRESHOLD = 10 + +GUIDED_OPTIONS = ('urls', 'hostnames', 'domains', 'crawl', 'from_ts', 'to', 'languages', 'limit') + + +def confirm_cost(n_crawls: Optional[int], engine: str, confirmed: bool) -> None: + """Prompt before running a potentially expensive index scan. + + `n_crawls` is the number of crawls the scan is restricted to, None if it is not restricted (or it cannot be + verified, e.g. for raw SQL). A scan bounded to <= LARGE_CRAWL_SET_THRESHOLD crawls runs without a prompt. + Otherwise we confirm interactively, abort in non-interactive sessions, unless `confirmed` (--yes).""" + if confirmed: + return + + if n_crawls is not None and n_crawls <= LARGE_CRAWL_SET_THRESHOLD: + return + + if n_crawls is None: + reason = (f'This {engine} query is not restricted to specific crawls (or uses custom SQL ' + 'whose crawl partition pruning could not be verified) and may scan ALL crawls.') + else: + reason = (f'This {engine} query is restricted to {n_crawls} crawls (more than ' + f'{LARGE_CRAWL_SET_THRESHOLD}) and may scan a large amount of data.') + + if not sys.stdin.isatty(): + raise SystemExit( + reason + ' Refusing to run a potentially expensive scan in non-interactive mode. ' + f'Restrict with --crawl (<={LARGE_CRAWL_SET_THRESHOLD} crawls), or pass --yes.' + ) + + LOGGER.warning(reason) + # the prompt goes to stderr, stdout may be the output of the captures + sys.stderr.write('Index SQL scans can be expensive (Athena bills per TB scanned). Proceed? [y/N] ') + sys.stderr.flush() + answer = input() + if answer.strip().lower() not in ('y', 'yes'): + raise SystemExit('Aborted by user.') + + +def normalize_row(row: Mapping) -> dict: + """Columnar index row -> record with CDX field names; timestamps as YYYYMMDDhhmmss strings. + + NULL values stay None, so that every record has all columns (see IndexBackend.fixed_schema).""" + record = {} + for column, value in row.items(): + if value is None: + pass + elif isinstance(value, datetime.datetime): + if value.tzinfo is not None: + value = value.astimezone(datetime.timezone.utc) + value = value.strftime('%Y%m%d%H%M%S') + elif column == 'fetch_time': + value = re.sub(r'\D', '', str(value))[:14] # Athena: '2024-07-13 10:22:33.000' + elif isinstance(value, (int, float)) and not isinstance(value, bool): + value = str(value) # the CDX API returns strings + record[COLUMNS.get(column, column)] = value + return record + + +class SqlBackend(IndexBackend): + """Common logic of the SQL backends: guided query building, cost guard, row normalization.""" + + engine = '' + fixed_schema = True + supported_options = frozenset({ + 'urls', 'hostnames', 'domains', 'crawl', 'from_ts', 'to', 'languages', 'limit', 'sql', 'url_fgrep', + 'url_fgrepv', + }) + + def __init__(self, location: str, assume_yes: bool = False): + self.location = location + self.assume_yes = assume_yes + + url_file_hint = '' + + def validate(self, query: IndexQuery) -> None: + if query.url_file or query.url_column or query.url_file_type: + raise UsageError( + f'--url-file cannot be used with the {self.name} index; instead, use a custom --sql query that joins ' + f'the index with a table of the target URLs, e.g.{self.url_file_hint}' + ) + super().validate(query) + if query.sql: + combined = [field_flag(option) for option in GUIDED_OPTIONS if option in query.options_used()] + if combined: + raise UsageError(f'{", ".join(combined)} cannot be combined with --sql') + elif not (query.urls or query.hostnames or query.domains): + raise UsageError(f'the {self.name} index needs --url, --hostname, --domain or --sql') + else: + url_surtkeys(query) # check --url + query_conditions(query) # check --from, --to and --language + + @abstractmethod + def from_clause(self, crawls: Optional[List[str]]) -> Tuple[str, bool]: + """FROM expression for the guided query, and whether it already prunes the crawls.""" + raise NotImplementedError # pragma: no cover + + @abstractmethod + def iter_sql_rows(self, sql: str) -> Iterator[dict]: + """Run the SQL and yield the result rows as dicts (column name -> value).""" + raise NotImplementedError # pragma: no cover + + def plan(self, query: IndexQuery, order_by: bool = True) -> 'SqlPlan': + """The SQL to run (--sql, or the guided query) and the number of crawls it is restricted to.""" + if query.sql: + return SqlPlan(self.name, self.location, query.sql.strip().rstrip(';'), n_crawls=None) + crawls = resolve_crawl_names(query.crawl) if query.crawl else None + from_clause, prunes_crawls = self.from_clause(crawls) + sql = build_sql( + query, from_clause, crawls=None if prunes_crawls else crawls, order_by=order_by, columns=list(COLUMNS), + ) + return SqlPlan(self.name, self.location, sql, n_crawls=len(crawls) if crawls else None) + + def select(self, query: IndexQuery) -> Iterator[Mapping]: + plan = self.plan(query) + confirm_cost(plan.n_crawls, self.engine, self.assume_yes) + for row in self.iter_sql_rows(plan.sql): + yield normalize_row(row) + + def select_count(self, query: IndexQuery) -> int: + plan = self.plan(query, order_by=False) + confirm_cost(plan.n_crawls, self.engine, self.assume_yes) + row = next(self.iter_sql_rows(f'SELECT count(*) AS n FROM ({plan.sql}) AS q')) + return int(row['n']) + + +@dataclass +class SqlPlan(QueryPlan): + """The SQL of a query and the number of crawls it is restricted to (None: all crawls, or unknown for --sql).""" + + backend_name: str + location: str + sql: str + n_crawls: Optional[int] + + def describe(self) -> str: + scope = f'{self.n_crawls} crawl(s)' if self.n_crawls else 'all crawls (or not verifiable)' + return f'-- {self.backend_name} query on {self.location}, restricted to {scope}\n{self.sql.strip()}\n' diff --git a/cdx_toolkit/indexes/backends/sql_builder.py b/cdx_toolkit/indexes/backends/sql_builder.py new file mode 100644 index 0000000..556a1d3 --- /dev/null +++ b/cdx_toolkit/indexes/backends/sql_builder.py @@ -0,0 +1,258 @@ +''' +SQL text for the columnar URL index (Athena, DuckDB): escaping, the WHERE clause of the guided queries, crawls. +''' +import datetime +import logging +import re +from typing import List, Optional, Tuple + +from cdx_toolkit.commoncrawl import normalize_crawl, get_cc_endpoints, match_cc_crawls +from cdx_toolkit.timeutils import timestamp_range + +from ..base import UsageError +from ..patterns import domain_to_surts, normalize_host, url_pattern_to_surts +from ..query import IndexQuery + + +logger = logging.getLogger(__name__) + +# Hostnames, TLDs and crawl names only ever contain these characters. Validating +# against this set both prevents SQL injection and catches malformed input early. +_SQL_LITERAL_RE = re.compile(r'^[A-Za-z0-9.\-]+$') + +# Default Common Crawl index mirror used to resolve --crawl to concrete crawl names. +CC_INDEX_MIRROR = 'https://index.commoncrawl.org/' + + +def escape_sql_literal(value: str) -> str: + """Validate and quote a value for safe inclusion in a SQL string literal. + + Only hostname/TLD/crawl-name characters (letters, digits, dot, hyphen) are + allowed, so the result cannot break out of the quotes or inject SQL.""" + if not isinstance(value, str) or not _SQL_LITERAL_RE.match(value): + raise ValueError( + f'Invalid value for SQL query literal: {value!r} ' + '(allowed characters: letters, digits, dot, hyphen)' + ) + return "'" + value + "'" + + +def quote_sql_string(value: str) -> str: + """Quote any string for a SQL string literal (single quotes are doubled).""" + if any(ord(c) < 32 for c in value): + raise ValueError(f'Invalid value for SQL query literal: {value!r} (control characters)') + return "'" + value.replace("'", "''") + "'" + + +def sql_like_prefix(prefix: str) -> str: + """LIKE pattern (with ESCAPE '\\') matching all strings that start with `prefix`.""" + escaped = prefix.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_') + return f"LIKE {quote_sql_string(escaped + '%')} ESCAPE '\\'" + + +def surt_tld(surt: str) -> str: + """'com,example)/path' -> 'com'""" + return re.split('[,):]', surt, maxsplit=1)[0] + + +def reversed_host(host: str) -> Optional[str]: + """The url_host_name_reversed of a host (e.g. www.example.com -> com.example.www), None for an IP address or + a host with other characters than letters, digits, dot and hyphen (these get no url_host_name_reversed + predicates).""" + labels = host.split('.') + if not _SQL_LITERAL_RE.match(host) or not all(labels) or all(label.isdigit() for label in labels): + return None + return '.'.join(reversed(labels)) + + +def reversed_host_range(reversed_prefix: str) -> str: + """url_host_name_reversed values starting with the reversed host: the host and its subdomains (and a few more + hosts, e.g. com.example-shop for com.example). A range, not a LIKE: the engines skip the row groups outside it + by their min/max statistics (the index is sorted by url_surtkey, which follows url_host_name_reversed).""" + escape_sql_literal(reversed_prefix) # validates + return f"url_host_name_reversed >= '{reversed_prefix}' AND url_host_name_reversed < '{reversed_prefix}/'" + + +def surt_host_hint(surt: str) -> Optional[str]: + """The url_host_name_reversed range of the hosts of a SURT key or prefix (e.g. 'com,example)/path' or + 'com,example,' or 'com,example:8080)/'), None if it has no plain host (an IP address).""" + host = re.split('[):]', surt, maxsplit=1)[0].rstrip(',') + reversed_prefix = reversed_host('.'.join(reversed(host.split(',')))) if host else None + return reversed_host_range(reversed_prefix) if reversed_prefix else None + + +def hostname_predicate(hostname: str) -> str: + """Captures of the exact host: url_host_name, and url_host_name_reversed for the min/max statistics (see + reversed_host_range).""" + predicate = f'url_host_name = {escape_sql_literal(hostname)}' + reversed_name = reversed_host(hostname) + if reversed_name is None: + return predicate + return f"(url_host_name_reversed = '{reversed_name}' AND {predicate})" + + +def domain_predicate(domain: str) -> str: + """Captures of the domain and its subdomains (like --domain for CDX files), e.g. for blog.example.com: + + url_host_name_reversed >= 'com.example.blog' AND url_host_name_reversed < 'com.example.blog/' + AND (url_host_name_reversed = 'com.example.blog' OR url_host_name_reversed LIKE 'com.example.blog.%') + + The range lets the engines skip the row groups of other hosts (see reversed_host_range), the small + dictionary-encoded url_host_name_reversed gives the exact host boundary. An IP address is matched by its + url_surtkey prefixes and registered domain instead.""" + surts = domain_to_surts(domain) + labels = surts[0][:-1].split(',')[::-1] # canonical, e.g. without www. + reversed_domain = reversed_host('.'.join(labels)) + if reversed_domain is not None: + return (f'({reversed_host_range(reversed_domain)} AND (url_host_name_reversed = \'{reversed_domain}\' ' + f"OR url_host_name_reversed LIKE '{reversed_domain}.%'))") + suffixes = ['.'.join(labels[i:]) for i in range(len(labels) - 1)] or labels + registered = ', '.join(escape_sql_literal(suffix) for suffix in suffixes) + surt_match = ' OR '.join(f'url_surtkey {sql_like_prefix(surt)}' for surt in surts) + return f'(url_host_registered_domain IN ({registered}) AND ({surt_match}))' + + +def surt_predicate(surt: str, prefix: bool) -> str: + """Captures of a url_surtkey (prefix), with the url_host_name_reversed range of its host for the min/max + statistics (see surt_host_hint).""" + predicate = f'url_surtkey {sql_like_prefix(surt)}' if prefix else f'url_surtkey = {quote_sql_string(surt)}' + hint = surt_host_hint(surt) + return f'({hint} AND {predicate})' if hint else predicate + + +def build_where_sql(query: IndexQuery, crawls: Optional[List[str]] = None) -> str: + """Build the WHERE body (without the `WHERE` keyword) of a guided query, shared by all SQL engines. + + The URL selectors are OR-ed: --hostname matches `url_host_name` (exact host, e.g. www.example.com), + --domain the domain with its subdomains (see domain_predicate), --url the `url_surtkey` (SURT prefixes or + exact keys). At least one of them is required. If `crawls` is a non-empty list of crawl names (e.g. + ['CC-MAIN-2025-33']), a `crawl IN (...)` partition filter is added -- the main lever for reducing scan cost. + --from, --to and --language are AND-ed (see query_conditions). Engines differ only in their FROM clause (see + build_sql).""" + hostnames = [normalize_host(h) for h in query.hostnames] + domains = [normalize_host(d) for d in query.domains] + surt_prefixes, surts = url_surtkeys(query) + if not hostnames and not domains and not surt_prefixes and not surts: + raise ValueError('an index query requires at least one hostname, registered domain or URL') + + tlds = {v.split('.')[-1] for v in hostnames + domains} + tlds |= {surt_tld(s) for s in surt_prefixes + surts} + query_tlds = ' OR '.join(f'url_host_tld = {escape_sql_literal(t)}' for t in sorted(tlds)) + + host_predicates = [hostname_predicate(h) for h in hostnames] + host_predicates += [domain_predicate(d) for d in domains] + host_predicates += [surt_predicate(s, prefix=False) for s in surts] + host_predicates += [surt_predicate(s, prefix=True) for s in surt_prefixes] + query_hosts = ' OR '.join(host_predicates) + + clauses = [ + "subset = 'warc'", + f'({query_tlds}) -- help the query optimizer', + f'({query_hosts})', + ] + if crawls: + crawl_in = ', '.join(escape_sql_literal(c) for c in crawls) + clauses.append(f'crawl IN ({crawl_in})') + clauses.extend(query_conditions(query)) + + return '\n AND '.join(clauses) + + +# Sorting range jobs by (warc_filename, warc_record_offset) groups records of the +# same WARC file with ascending offsets, which improves S3 range-read locality (and +# enables coalescing adjacent ranges) at fetch time. +ORDER_BY_SQL = 'ORDER BY warc_filename, warc_record_offset' + + +def build_sql( + query: IndexQuery, + from_clause: str, + crawls: Optional[List[str]] = None, + order_by: bool = True, + columns: Optional[List[str]] = None, +) -> str: + """The SELECT of a guided query (--url, --hostname, --domain, ...) of the columnar index. + + `from_clause` is the text following FROM (e.g. `ccindex` for Athena, or a `read_parquet(...)` expression for + DuckDB); `crawls` adds a partition filter (see build_where_sql). When `order_by` is set, results are sorted by + (warc_filename, warc_record_offset) for fetch-time read locality. `columns` defaults to the WARC location + columns.""" + where_sql = build_where_sql(query, crawls) + order_sql = f'\n {ORDER_BY_SQL}' if order_by else '' + limit_sql = f'\n LIMIT {query.limit}' if query.limit and query.limit > 0 else '' + columns_sql = ', '.join(columns or ('warc_filename', 'warc_record_offset', 'warc_record_length')) + + return f""" + SELECT + {columns_sql} + FROM {from_clause} + WHERE {where_sql}{order_sql}{limit_sql}""" + + +def endpoint_to_crawl_name(endpoint: str) -> str: + """Turn a collinfo cdx-api endpoint into its crawl name. + + e.g. 'https://index.commoncrawl.org/CC-MAIN-2025-33-index' -> 'CC-MAIN-2025-33'.""" + name = endpoint.rstrip('/').split('/')[-1] + if name.endswith('-index'): + name = name[:-len('-index')] + return name + + +def resolve_crawl_names(crawl_arg) -> List[str]: + """Resolve a --crawl value to concrete CC-MAIN crawl names for the partition filter. + + Reuses the CDX path's helpers so `--crawl` accepts the same forms (comma-separated + names or an integer for the most recent N crawls).""" + crawls = normalize_crawl([crawl_arg]) + raw_index_list = get_cc_endpoints(CC_INDEX_MIRROR) + matched = match_cc_crawls(crawls, raw_index_list) + return [endpoint_to_crawl_name(ep) for ep in matched] + + +def sql_timestamp(timestamp: str, end: bool = False) -> str: + """CDX timestamp (e.g. 2008, 200906302359) -> SQL TIMESTAMP literal of its first second, or with `end`, of the + second after its last second (an exclusive end, which includes the fractions of the last second).""" + try: + first, last = timestamp_range(timestamp) + moment = last + datetime.timedelta(seconds=1) if end else first + except ValueError as e: + raise UsageError(str(e)) from None + except OverflowError: # the end of 9999 + raise UsageError(f'invalid timestamp {timestamp!r} (too late)') from None + return f"TIMESTAMP '{moment:%Y-%m-%d %H:%M:%S}'" + + +def check_language(code: str) -> str: + """An ISO 639-3 language code (three letters, e.g. eng or deu), in lowercase.""" + language = code.strip().lower() + if not re.fullmatch(r'[a-z]{3}', language): + raise UsageError(f'invalid language {code!r}, use ISO 639-3 codes (three letters), e.g. eng or deu') + return language + + +def url_surtkeys(query: IndexQuery) -> Tuple[List[str], List[str]]: + """--url patterns -> (url_surtkey prefixes, exact url_surtkey values).""" + prefixes, exact = [], [] + for pattern in query.urls: + p, e = url_pattern_to_surts(pattern) + prefixes += p + exact += e + return prefixes, exact + + +def query_conditions(query: IndexQuery) -> List[str]: + """WHERE predicates for --from, --to and --language.""" + conditions = [] + if query.from_ts: + conditions.append(f'fetch_time >= {sql_timestamp(query.from_ts)}') + if query.to: + conditions.append(f'fetch_time < {sql_timestamp(query.to, end=True)}') + if query.languages: + # primary content language: the first of the comma-separated content_languages + predicates = [] + for code in query.languages: + language = check_language(code) + predicates += [f"content_languages = '{language}'", f"content_languages LIKE '{language},%'"] + conditions.append('(' + ' OR '.join(predicates) + ')') + return conditions diff --git a/cdx_toolkit/indexes/base.py b/cdx_toolkit/indexes/base.py new file mode 100644 index 0000000..7259f0d --- /dev/null +++ b/cdx_toolkit/indexes/base.py @@ -0,0 +1,106 @@ +from abc import ABC, abstractmethod +from itertools import islice +import logging +from typing import FrozenSet, Iterable, Iterator, List, Mapping + +from cdx_toolkit.timeutils import timestamp_range + +from .query import IndexQuery, field_flag + + +LOGGER = logging.getLogger(__name__) + + +class UsageError(ValueError): + """Invalid query or options (the command line reports it without a traceback).""" + + +class QueryPlan(ABC): + """A query in the form of one backend, e.g. SQL text or SURT prefixes; what --dry-run shows.""" + + @abstractmethod + def describe(self) -> str: + raise NotImplementedError # pragma: no cover + + +class IndexBackend(ABC): + """An index that can be queried for captures: the CDX API, CDX files, or the columnar index (Athena, DuckDB). + + A query (IndexQuery, the same for every backend) becomes the backend's plan (plan()), which is run by + select() and select_count(), or shown by explain(). The public methods records(), count() and explain() + validate the query first.""" + + name: str = '' + supported_options: FrozenSet[str] = frozenset() + # True if all records have the same fields (in the same order), e.g. the columns of a SQL result + fixed_schema: bool = False + + def validate(self, query: IndexQuery) -> None: + unsupported = sorted(query.options_used() - self.supported_options) + if unsupported: + options = ', '.join(field_flag(option) for option in unsupported) + raise UsageError(f'{options} cannot be used with the {self.name} index') + if query.limit is not None and query.limit < 1: + raise UsageError(f'--limit must be at least 1, not {query.limit}') + for option, timestamp in (('--from', query.from_ts), ('--to', query.to), ('--closest', query.closest)): + if timestamp is not None: + try: + timestamp_range(timestamp) + except ValueError as e: + raise UsageError(f'{option}: {e}') from None + + def input_locations(self, query: IndexQuery) -> List[str]: + """The files (paths, directories or globs) the query reads, which an output must not overwrite.""" + return [] + + @abstractmethod + def plan(self, query: IndexQuery) -> QueryPlan: + """The query in the form of this backend.""" + raise NotImplementedError # pragma: no cover + + @abstractmethod + def select(self, query: IndexQuery) -> Iterator[Mapping]: + """Yield the selected captures as mappings with CDX field names (url, timestamp, filename, ...). + + A backend can push --limit down to the index, but need not stop at it (see records()).""" + raise NotImplementedError # pragma: no cover + + @abstractmethod + def select_count(self, query: IndexQuery) -> int: + """Cheap (possibly estimated) number of captures matching the query, at most --limit (see count()).""" + raise NotImplementedError # pragma: no cover + + def records(self, query: IndexQuery) -> Iterator[Mapping]: + """The captures of the query: at most --limit, then filtered by --url-fgrep/--url-fgrepv (client-side).""" + self.validate(query) + return filter_by_url(islice(self.select(query), query.limit), query.url_fgrep, query.url_fgrepv) + + def count(self, query: IndexQuery) -> int: + """The number of captures matching the query, at most --limit; without the client-side + --url-fgrep/--url-fgrepv, which cannot be combined with it.""" + self.validate(query) + if query.url_fgrep or query.url_fgrepv: + raise UsageError('--count cannot be used with --url-fgrep or --url-fgrepv') + return self.select_count(query) + + def explain(self, query: IndexQuery) -> str: + """Describe what would be queried (--dry-run), without querying the index.""" + self.validate(query) + return self.plan(query).describe() + + +def filter_by_url(records: Iterable[Mapping], url_fgrep=None, url_fgrepv=None) -> Iterator[Mapping]: + if not (url_fgrep or url_fgrepv): + yield from records # records need no url, e.g. of a --sql query that selects only the WARC location + return + for record in records: + url = record.get('url') + if url is None: + raise UsageError('--url-fgrep and --url-fgrepv need captures with a url (e.g. SELECT url in --sql)') + if url_fgrep and url_fgrep not in url: + LOGGER.debug('skipping due to --url-fgrep: %s', url) + continue + if url_fgrepv and url_fgrepv in url: + LOGGER.debug('skipping due to --url-fgrepv: %s', url) + continue + yield record diff --git a/cdx_toolkit/indexes/patterns.py b/cdx_toolkit/indexes/patterns.py new file mode 100644 index 0000000..f502a0d --- /dev/null +++ b/cdx_toolkit/indexes/patterns.py @@ -0,0 +1,82 @@ +''' +URL patterns (as for the CDX API) -> SURT keys and prefixes, used to select captures from CDX files +and from the columnar index (url_surtkey). +''' +from typing import List, Tuple + +from .base import UsageError + + +def to_surt(url: str) -> str: + from url_is_in import convert_url_to_surt + + if '://' not in url: + url = 'http://' + url # without a scheme, a port is not recognized ('example.com:8080/') + return convert_url_to_surt(url) + + +def url_pattern_to_surts(pattern: str) -> Tuple[List[str], List[str]]: + """A CDX API URL pattern -> (SURT prefixes, exact SURT keys). + + - `*.example.com`: the domain and its subdomains (with any port) + - `example.com/*`, `example.com/path*`: all URLs starting with the pattern; a trailing / or ? is kept as a + boundary (`example.com/blog/*` does not select `/blogger` or `/blog?x`), like pywb's prefix search + - `example.com/path`: exactly this URL + """ + pattern = pattern.strip() + if pattern.startswith('*.'): + domain = pattern[2:] + if not domain or any(c in domain for c in '*/?'): + raise UsageError(f'invalid URL pattern {pattern!r}, a domain pattern is e.g. *.example.com') + base = to_surt(domain)[:-2] # 'com,example)/' -> 'com,example' + return [base + ')', base + ',', base + ':'], [] # the domain, its subdomains, with a port + if pattern.endswith('*'): + prefix = pattern[:-1] + if '*' in prefix: + raise UsageError(f'invalid URL pattern {pattern!r}, * can only be at the start or the end') + key = to_surt(prefix) + # the SURT drops a trailing / or ? ('example.com/blog/' -> 'com,example)/blog'), the prefix keeps it + for boundary in ('/', '?'): + if prefix.endswith(boundary) and not key.endswith(boundary): + key += boundary + return [key], [] + if '*' in pattern: + raise UsageError(f'invalid URL pattern {pattern!r}, * can only be at the start or the end') + return [], [to_surt(pattern)] + + +def normalize_host(host: str) -> str: + """A hostname or domain as in the index (url_host_name of the columnar index): lower case, no trailing dot.""" + return host.strip().lower().rstrip('.') + + +def url_hostname(url: str) -> str: + """The (normalized) hostname of a URL, '' if it has none.""" + from urllib.parse import urlsplit + + if '://' not in url: + url = 'http://' + url + return normalize_host(urlsplit(url).hostname or '') + + +def hostname_to_surts(hostname: str) -> List[str]: + """The URLs of this host, and of the host with or without www. (the SURT key drops www.), with any port; the + exact host is checked with url_hostname().""" + key = to_surt(normalize_host(hostname) + '/') # 'com,example)/' + return [key, key[:-2] + ':'] + + +def domain_to_surts(domain: str) -> List[str]: + """All URLs of the domain and its subdomains.""" + return url_pattern_to_surts('*.' + normalize_host(domain))[0] + + +def domain_entry_to_surts(entry: str) -> List[str]: + """A --url-file-type domain entry: a domain with its subdomains, or a URL prefix if it has a path.""" + entry = entry.strip() + host_and_path = entry.split('://', 1)[-1] + if '*' in entry: + raise UsageError(f'invalid domain entry {entry!r} (no wildcards with --url-file-type domain)') + if '/' not in host_and_path.rstrip('/') and '?' not in host_and_path: + return domain_to_surts(host_and_path.rstrip('/')) + return [to_surt(entry)] diff --git a/cdx_toolkit/indexes/query.py b/cdx_toolkit/indexes/query.py new file mode 100644 index 0000000..c983dd7 --- /dev/null +++ b/cdx_toolkit/indexes/query.py @@ -0,0 +1,64 @@ +from dataclasses import dataclass, field, fields +from typing import List, Optional, Set, Tuple + + +# command line flags of the query fields, if they differ from -- +FIELD_FLAGS = { + 'urls': '--url', + 'hostnames': '--hostname', + 'domains': '--domain', + 'from_ts': '--from', + 'filters': '--filter', + 'languages': '--language', +} + + +LIST_FIELDS = ('urls', 'hostnames', 'domains', 'filters', 'languages') + + +def field_flag(name: str) -> str: + return FIELD_FLAGS.get(name, '--' + name.replace('_', '-')) + + +@dataclass +class IndexQuery: + """What to select from an index, independent of the index backend. + + URL selectors (urls, hostnames, domains, url_file) are OR-ed; all other options are AND-ed. + `sql` is a complete query for SQL backends, used instead of the other selectors. + """ + + urls: List[str] = field(default_factory=list) + hostnames: List[str] = field(default_factory=list) + domains: List[str] = field(default_factory=list) + url_file: Optional[str] = None + url_column: Optional[str] = None + url_file_type: Optional[str] = None # 'pattern' (default) or 'domain' + crawl: Optional[str] = None + from_ts: Optional[str] = None + to: Optional[str] = None + closest: Optional[str] = None + filters: List[str] = field(default_factory=list) + url_fgrep: Optional[str] = None + url_fgrepv: Optional[str] = None + limit: Optional[int] = None + languages: List[str] = field(default_factory=list) + sql: Optional[str] = None + + def __post_init__(self): + # a single value, e.g. IndexQuery(urls='example.com/*'), would otherwise be iterated by character + for name in LIST_FIELDS: + if isinstance(getattr(self, name), str): + setattr(self, name, [getattr(self, name)]) + + def url_patterns(self) -> List[Tuple[str, Optional[str]]]: + """All URL selectors as (pattern, CDX API matchType) pairs.""" + return ( + [(url, None) for url in self.urls] + + [(hostname, 'host') for hostname in self.hostnames] + + [(domain, 'domain') for domain in self.domains] + ) + + def options_used(self) -> Set[str]: + """Names of the options that are set, used to check what a backend supports.""" + return {f.name for f in fields(self) if getattr(self, f.name) not in (None, [])} diff --git a/cdx_toolkit/indexes/resolve.py b/cdx_toolkit/indexes/resolve.py new file mode 100644 index 0000000..1da1e1e --- /dev/null +++ b/cdx_toolkit/indexes/resolve.py @@ -0,0 +1,95 @@ +from typing import Optional +from urllib.parse import urlparse + +from .base import IndexBackend, UsageError +from .backends.cdx_api import CdxApiBackend + + +# index names that map to CDX API sources known by CDXFetcher +INDEX_NAMES = ('cc', 'ia') + +# index names that map to locations of Common Crawl's columnar URL index (queried with DuckDB) +INDEX_ALIASES = { + 'cc-url-index-s3': 's3://commoncrawl/cc-index/table/cc-main/warc/', + 'cc-url-index-hf': 'hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/', +} + +INDEX_BACKENDS = ('auto', 'cdx-api', 'cdx-files', 'duckdb', 'athena') + +GLOB_CHARS = ('*', '?', '[') + + +def is_cdx_api_location(location: str) -> bool: + """An index name, or an http(s) URL whose last path segment has no file extension.""" + if location in INDEX_NAMES: + return True + parsed = urlparse(location) + if parsed.scheme not in ('http', 'https'): + return False + if any(c in parsed.path for c in GLOB_CHARS): + return False + last_segment = parsed.path.rstrip('/').rsplit('/', 1)[-1] + return '.' not in last_segment + + +def detect_index_backend(location: str) -> str: + """The backend of an index, from its location only (no network access).""" + if location in INDEX_ALIASES or location.endswith('.parquet'): + return 'duckdb' + if location.startswith('athena://'): + return 'athena' + if is_cdx_api_location(location): + return 'cdx-api' + return 'cdx-files' + + +def open_index( + location: str, + backend: str = 'auto', + cc_mirror: Optional[str] = None, + cc_sort: Optional[str] = None, + assume_yes: bool = False, +) -> IndexBackend: + """Map an index name or location to the backend that can query it. + + - 'cc', 'ia', or http(s) URLs without a file extension -> CDX API + - 'athena://...' -> Athena + - 'cc-url-index-s3', 'cc-url-index-hf', '*.parquet', or any path with backend 'duckdb' -> DuckDB + - anything else (local or remote CDX files, globs) -> CDX files + + `backend` ('cdx-api', 'cdx-files', 'duckdb' or 'athena') overrides the detection, e.g. 'cdx-files' for a CDX + file at an http(s) URL without file extension. `cc_mirror` and `cc_sort` are options of the Common Crawl CDX + API ('cc') and rejected for other indexes; `assume_yes` is accepted by all (only SQL indexes ask). + """ + if backend not in INDEX_BACKENDS: + raise UsageError(f'unknown index backend {backend!r} (choose from {", ".join(INDEX_BACKENDS)})') + if backend == 'auto': + backend = detect_index_backend(location) + if (cc_mirror is not None or cc_sort is not None) and not (backend == 'cdx-api' and location == 'cc'): + options = ' and '.join(name for name, value in (('--cc-mirror', cc_mirror), ('--cc-sort', cc_sort)) + if value is not None) + raise UsageError(f'{options} can only be used with the Common Crawl CDX API (cc), not with {location}') + location = INDEX_ALIASES.get(location, location) + + if backend == 'athena': + from .backends.athena import AthenaBackend + + return AthenaBackend(location, assume_yes=assume_yes) + + if backend == 'duckdb': + if location in INDEX_NAMES or is_cdx_api_location(location) or location.startswith('athena://'): + raise UsageError(f'{location} cannot be queried with DuckDB; use a path to the parquet index') + from .backends.duckdb import DuckDbBackend + + return DuckDbBackend(location, assume_yes=assume_yes) + + if backend == 'cdx-api': + if location not in INDEX_NAMES and urlparse(location).scheme not in ('http', 'https'): + raise UsageError(f'{location} is not a CDX API; use cc, ia or the http(s) URL of a CDX API server') + return CdxApiBackend(location, cc_mirror=cc_mirror, cc_sort=cc_sort) + + if location in INDEX_NAMES or location.startswith('athena://'): + raise UsageError(f'{location} is not a path of CDX files') + from .backends.cdx_files import CdxFilesBackend + + return CdxFilesBackend(location) diff --git a/cdx_toolkit/myrequests.py b/cdx_toolkit/myrequests.py index 6e8d684..85487ec 100644 --- a/cdx_toolkit/myrequests.py +++ b/cdx_toolkit/myrequests.py @@ -1,6 +1,7 @@ -from typing import Optional +from typing import Dict, Optional import requests import logging +import threading import time from urllib.parse import urlparse @@ -23,44 +24,73 @@ } -def dns_fatal(hostname): - """We have a dns error, should we fail immediately or not?""" - if hostname not in previously_seen_hostnames: - return True - - -retry_info = { - 'default': { - 'next_fetch': 0, - 'minimum_interval': DEFAULT_MIN_RETRY_INTERVAL, - }, - 'index.commoncrawl.org': { - 'next_fetch': 0, - 'minimum_interval': CC_INDEX_MIN_RETRY_INTERVAL, - }, - 'data.commoncrawl.org': { - 'next_fetch': 0, - 'minimum_interval': CC_DATA_MIN_RETRY_INTERVAL, - }, - 'web.archive.org': { - 'next_fetch': 0, - 'minimum_interval': IA_MIN_RETRY_INTERVAL, - }, -} +# A CDX index server (pywb) that cannot read its index (e.g. from S3) answers with the text of the exception, e.g. +# "Error: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))", and a +# status that is not retried otherwise: a temporary failure +TEMPORARY_CDX_ERRORS = ( + 'Connection aborted', 'RemoteDisconnected', 'Connection reset', 'Read timed out', 'ConnectTimeout', + 'Max retries exceeded', +) + + +def is_temporary_cdx_error(resp) -> bool: + text = resp.text[:1000] + return text.startswith('Error:') and any(marker in text for marker in TEMPORARY_CDX_ERRORS) + + +def dns_fatal(hostname) -> bool: + """We have a dns error, should we fail immediately or not? Not for a host that worked before (the error may be + temporary).""" + return hostname not in previously_seen_hostnames + + +class HostRateLimiter: + """The minimum interval between the starts of requests to the same host, also when requests are made from + several threads (e.g. the parallel WARC readers of cdxt repackage). Every attempt, also a retry, takes its + turn. Times are time.monotonic().""" + + def __init__(self, default_interval: float, intervals: Dict[str, float]): + self.default_interval = default_interval + self.intervals = dict(intervals) # seconds, by hostname + self._next_fetch: Dict[str, float] = {} # the earliest start of the next request, by hostname + self._lock = threading.Lock() + + def minimum_interval(self, hostname: str) -> float: + return self.intervals.get(hostname, self.default_interval) + + def reserve(self, hostname: str, now: float) -> float: + """The time at which the next request to `hostname` may start, at least the minimum interval after the + previous one; the time is reserved, so concurrent requests are spaced out too.""" + with self._lock: + fetch_time = max(now, self._next_fetch.get(hostname, now)) + self._next_fetch[hostname] = fetch_time + self.minimum_interval(hostname) + return fetch_time + + def finish(self, hostname: str) -> None: + """After a request (which may have taken long, with retries): the next one may start the minimum interval + from now, or later if that time is reserved already.""" + with self._lock: + next_fetch = time.monotonic() + self.minimum_interval(hostname) + self._next_fetch[hostname] = max(self._next_fetch.get(hostname, 0), next_fetch) -def get_retries(hostname): - if hostname not in retry_info: - retry_info[hostname] = retry_info['default'].copy() - LOGGER.debug('initializing retry info for new host ' + hostname) - entry = retry_info[hostname] - if not entry['next_fetch']: - entry['next_fetch'] = time.time() - return entry['next_fetch'], entry['minimum_interval'] +RATE_LIMITER = HostRateLimiter(DEFAULT_MIN_RETRY_INTERVAL, { + 'index.commoncrawl.org': CC_INDEX_MIN_RETRY_INTERVAL, + 'data.commoncrawl.org': CC_DATA_MIN_RETRY_INTERVAL, + 'web.archive.org': IA_MIN_RETRY_INTERVAL, +}) -def update_next_fetch(hostname, next_fetch): - retry_info[hostname]['next_fetch'] = next_fetch +class RequestCancelled(Exception): + """The request was cancelled with its `stop` event (while waiting before the request or a retry).""" + + +def _sleep(seconds: float, stop: Optional[threading.Event]) -> None: + """Sleep, or raise RequestCancelled as soon as `stop` is set.""" + if stop is None: + time.sleep(seconds) + elif stop.wait(seconds): + raise RequestCancelled() def myrequests_get( @@ -72,9 +102,12 @@ def myrequests_get( raise_error_after_n_errors: Optional[int] = None, raise_warning_after_n_errors: Optional[int] = None, retry_max_sec: int = 60, + stop: Optional[threading.Event] = None, ): - t = time.time() + """GET with the minimum interval between requests to a host, and retries. + With `stop`, the waits (before the request, between retries) end with RequestCancelled once it is set; a + request in progress ends with its timeout.""" if raise_error_after_n_errors is None: raise_error_after_n_errors = MAX_ERRORS @@ -82,15 +115,7 @@ def myrequests_get( raise_warning_after_n_errors = WARNING_AFTER_N_ERRORS hostname = urlparse(url).hostname - next_fetch, minimum_interval = get_retries(hostname) - - if t < next_fetch: - dt = next_fetch - t - if dt > 3.1: - LOGGER.debug('sleeping for {:.3f}s before next fetch'.format(dt)) - time.sleep(dt) - # next_fetch is also updated at the bottom - update_next_fetch(hostname, next_fetch + minimum_interval) + minimum_interval = RATE_LIMITER.minimum_interval(hostname) if params: if 'from_ts' in params: @@ -111,10 +136,14 @@ def myrequests_get( retries = 0 connect_errors = 0 while retry: + if stop is not None and stop.is_set(): + raise RequestCancelled() + _wait_for_turn(hostname, stop) # after the backoff of a retry: both the backoff and the interval hold try: LOGGER.debug('getting %s %r', url, params) resp = requests.get(url, params=params, headers=headers, timeout=(30.0, 30.0), allow_redirects=False) - if cdx and resp.status_code in {400, 404}: + temporary_cdx_error = cdx and is_temporary_cdx_error(resp) + if cdx and resp.status_code in {400, 404} and not temporary_cdx_error: # 400: ia html error page -- probably page= is too big -- not an error # 404: pywb {'error': 'No Captures found for: www.pbxxxxxxm.com/*'} -- not an error LOGGER.debug('giving up with status %d, no captures found', resp.status_code) @@ -123,7 +152,7 @@ def myrequests_get( if allow404 and resp.status_code == 404: retry = False break - if resp.status_code in {429, 500, 502, 503, 504, 509}: # pragma: no cover + if resp.status_code in {429, 500, 502, 503, 504, 509} or temporary_cdx_error: # 503=slow down, 50[24] are temporary outages, 500=Amazon S3 generic error # CC takes a 503 from storage and then emits a 500 with error text in resp.text # I have never seen IA or CC send 429 or 509, but just in case... @@ -133,7 +162,7 @@ def myrequests_get( LOGGER.log(level, 'retrying after %.2fs for %d', retry_sec, resp.status_code) if resp.text: LOGGER.log(level, 'response body is %s', resp.text) - time.sleep(retry_sec) + _sleep(retry_sec, stop) retry_sec = min(retry_sec * 2, retry_max_sec) continue if resp.status_code in {400, 404}: # pragma: no cover @@ -156,7 +185,7 @@ def myrequests_get( or ('nodename nor servname provided, or not known' in string) # macos or ('getaddrinfo failed' in string) ): # windows - if dns_fatal(url): + if dns_fatal(hostname): raise ValueError('invalid hostname in url ' + url) from None if connect_errors > raise_error_after_n_errors: @@ -165,17 +194,23 @@ def myrequests_get( if connect_errors > raise_warning_after_n_errors: LOGGER.warning(string) LOGGER.info('retrying after {:.2f}s for '.format(retry_max_sec) + str(e)) - time.sleep(retry_max_sec) # notice the extra-long sleep + _sleep(retry_max_sec, stop) # notice the extra-long sleep retry_sec = min(retry_sec * 2, retry_max_sec) except requests.exceptions.RequestException as e: # pragma: no cover LOGGER.warning('something unexpected happened, giving up after %s', str(e)) raise - hostname = urlparse(url).hostname - if hostname not in previously_seen_hostnames: - previously_seen_hostnames.add(hostname) - - # in case we had a lot of retries, etc - update_next_fetch(hostname, time.time() + minimum_interval) + previously_seen_hostnames.add(hostname) + RATE_LIMITER.finish(hostname) # in case we had a lot of retries, etc return resp + + +def _wait_for_turn(hostname: str, stop: Optional[threading.Event]) -> None: + """Wait until a request to `hostname` may start (see HostRateLimiter).""" + now = time.monotonic() + dt = RATE_LIMITER.reserve(hostname, now) - now + if dt > 0: + if dt > 3.1: + LOGGER.debug('sleeping for {:.3f}s before next fetch'.format(dt)) + _sleep(dt, stop) diff --git a/cdx_toolkit/record_io.py b/cdx_toolkit/record_io.py new file mode 100644 index 0000000..2a26e69 --- /dev/null +++ b/cdx_toolkit/record_io.py @@ -0,0 +1,341 @@ +''' +Reading and writing index records (captures) as JSONL, CDXJ, CSV, TSV or Parquet. + +Paths can be local or remote (via fsspec), optionally gzipped; "-" is stdin/stdout. +Parquet needs the optional pyarrow dependency (cdx_toolkit[parquet]) and a file path. +''' +import contextlib +import csv +import fnmatch +import itertools +import json +import os +import re +import sys +import tempfile +from typing import IO, Iterable, Iterator, List, Mapping, Optional, Tuple + +import fsspec +from fsspec.implementations.local import LocalFileSystem + + +TEXT_FORMATS = ('jsonl', 'cdxj', 'csv', 'tsv') +FORMATS = TEXT_FORMATS + ('parquet',) + +# Common Crawl's index files are named *.cdx(.gz) but use the CDXJ format (crawl index JSON) +FORMAT_ALIASES = {'cdx': 'cdxj'} +# formats for the command line, including the aliases +FORMAT_CHOICES = ('jsonl', 'cdx', 'cdxj', 'csv', 'tsv', 'parquet') + +# records per Parquet row group +PARQUET_BATCH_SIZE = 100_000 + +EXTENSION_FORMATS = { + '.jsonl': 'jsonl', + '.json': 'jsonl', + '.cdxj': 'cdxj', + '.cdx': 'cdxj', + '.csv': 'csv', + '.tsv': 'tsv', + '.parquet': 'parquet', +} + +# column names of the columnar index (and range-job CSVs) -> CDX field names +FIELD_ALIASES = { + 'warc_filename': 'filename', + 'warc_record_offset': 'offset', + 'warc_record_length': 'length', +} + +CDXJ_LINE = re.compile(r'^\S+ \d{4,14} \{') + + +def canonical_format(fmt: Optional[str]) -> Optional[str]: + """'cdx' -> 'cdxj'; other formats unchanged.""" + return FORMAT_ALIASES.get(fmt, fmt) + + +def format_from_path(path: str) -> Optional[str]: + """Infer the record format from a file extension (ignoring .gz), None if unknown.""" + if path == '-': + return None + path = path.lower() + if path.endswith('.gz'): + path = path[:-3] + for extension, fmt in EXTENSION_FORMATS.items(): + if path.endswith(extension): + return fmt + return None + + +def open_text(path: str, mode: str) -> IO: + """Open a local or remote text file; '-' means stdin/stdout (which is not closed).""" + if path == '-': + return contextlib.nullcontext(sys.stdout if 'w' in mode else sys.stdin) + return fsspec.open(path, mode + 't', compression='infer', newline='') + + +class RecordWriter: + """Write records in one of TEXT_FORMATS; `fields` selects and orders the fields (default: all). + + Fields with the value None are left out (JSONL, CDXJ) or empty (CSV, TSV). Without `fields`, the CSV/TSV + columns are the fields of the first record, and a later record with another field is an error (see + write_records, which finds all fields first).""" + + def __init__(self, fh: IO, fmt: str, fields: Optional[List[str]] = None): + fmt = canonical_format(fmt) + if fmt not in TEXT_FORMATS: + raise ValueError(f'unknown format {fmt!r}') + self.fh = fh + self.fmt = fmt + self.fields = fields + self._csv_writer = None + + def write(self, record: Mapping) -> None: + record = dict(record) + selected = {k: record[k] for k in self.fields if k in record} if self.fields else record + if self.fmt in ('jsonl', 'cdxj'): + selected = {k: v for k, v in selected.items() if v is not None} + + if self.fmt == 'jsonl': + self.fh.write(json.dumps(selected, sort_keys=True, default=str) + '\n') + elif self.fmt == 'cdxj': + for key in ('urlkey', 'timestamp'): + if key not in record: + raise ValueError(f'cannot write CDXJ: record has no {key!r} field') + data = {k: v for k, v in selected.items() if k not in ('urlkey', 'timestamp')} + self.fh.write(f'{record["urlkey"]} {record["timestamp"]} {json.dumps(data, default=str)}\n') + else: + if self._csv_writer is None: + self._csv_writer = csv.DictWriter( + self.fh, + fieldnames=self.fields or list(record.keys()), + delimiter='\t' if self.fmt == 'tsv' else ',', + lineterminator='\n', + extrasaction='ignore' if self.fields else 'raise', + ) + self._csv_writer.writeheader() + self._csv_writer.writerow(selected) + + +def is_glob(path: str) -> bool: + # in http(s) URLs, ? starts the query string + chars = '*[' if path.lower().startswith(('http://', 'https://')) else '*?[' + return path != '-' and any(c in path for c in chars) + + +def check_glob(path: str) -> None: + """Globs need a filesystem that can list files: not http(s).""" + if is_glob(path) and path.lower().startswith(('http://', 'https://')): + raise ValueError( + f'globs do not work with http(s) URLs ({path}), because they cannot list files; ' + 'use e.g. s3://commoncrawl/... (needs AWS credentials) or one URL per file' + ) + + +def overlaps(output: str, location: str) -> bool: + """Whether writing `output` can overwrite `location` (a file, a directory or a glob), or the other way around: + the same file (also through symlinks or hard links), a file in the directory, or a file matching the glob.""" + if output == '-' or location == '-': + return False + protocols = {fsspec.core.split_protocol(path)[0] or 'file' for path in (output, location)} + if len(protocols) > 1 and protocols != {'file', 'local'}: + return False + out_fs, out_path = fsspec.url_to_fs(output) + in_fs, in_path = fsspec.url_to_fs(location) + if type(out_fs) is not type(in_fs): + return False + if isinstance(out_fs, LocalFileSystem): + out_real = os.path.normcase(os.path.realpath(out_path)) + for path in (in_fs.glob(in_path) if is_glob(in_path) else [in_path]): + real = os.path.normcase(os.path.realpath(path)) + if out_real == real or out_real.startswith(real.rstrip(os.sep) + os.sep) \ + or real.startswith(out_real.rstrip(os.sep) + os.sep): + return True + if os.path.exists(out_path) and os.path.exists(path) and os.path.samefile(out_path, path): + return True + return False + # remote: compare the paths (a glob matches across '/', which rejects rather too much than too little) + out_path, in_path = out_path.rstrip('/'), in_path.rstrip('/') + return ( + out_path == in_path or out_path.startswith(in_path + '/') or in_path.startswith(out_path + '/') + or (is_glob(in_path) and fnmatch.fnmatchcase(out_path, in_path)) + ) + + +def is_empty_dir(path: str) -> bool: + """Whether a local or remote path is missing or an empty directory (for S3, a prefix without objects).""" + fs, fs_path = fsspec.url_to_fs(path) + try: + return not fs.ls(fs_path.rstrip('/'), detail=False) + except FileNotFoundError: + return True + except NotADirectoryError: + return False + + +def expand_paths(path: str) -> List[str]: + """A local or remote path, or all files matching a glob (e.g. s3://bucket/indexes/*.cdx.gz).""" + if not is_glob(path): + return [path] + check_glob(path) + fs, fs_path = fsspec.url_to_fs(path) + return [fs.unstrip_protocol(p) for p in sorted(fs.glob(fs_path))] + + +def read_records(path: str, fmt: Optional[str] = None) -> Iterator[dict]: + """Read records from a file, all files matching a glob, or stdin ('-'). + + The format is detected from the first line of each file if not given.""" + fmt = canonical_format(fmt) + paths = expand_paths(path) + if not paths: + raise FileNotFoundError(f'no files found for {path}') + for p in paths: + if fmt == 'parquet' or (fmt is None and format_from_path(p) == 'parquet'): + yield from read_parquet(p) + else: + yield from _read_records(p, fmt) + + +def _read_records(path: str, fmt: Optional[str] = None) -> Iterator[dict]: + with open_text(path, 'r') as fh: + lines = iter(fh) + first_line = next(lines, None) + if first_line is None: + return + lines = itertools.chain([first_line], lines) + fmt = fmt or sniff_format(first_line) + + if fmt == 'jsonl': + records = (json.loads(line) for line in lines if line.strip()) + elif fmt == 'cdxj': + records = (parse_cdxj_line(line) for line in lines if line.strip()) + elif fmt in ('csv', 'tsv'): + records = csv.DictReader(lines, delimiter='\t' if fmt == 'tsv' else ',') + else: + raise ValueError(f'unknown format {fmt!r}') + + for record in records: + yield normalize_field_names(record) + + +def sniff_format(line: str) -> str: + if line.lstrip().startswith('{'): + return 'jsonl' + if CDXJ_LINE.match(line): + return 'cdxj' + if '\t' in line: + return 'tsv' + return 'csv' + + +def parse_cdxj_line(line: str) -> dict: + urlkey, timestamp, data = line.split(' ', 2) + record = json.loads(data) + record.setdefault('urlkey', urlkey) + record.setdefault('timestamp', timestamp) + return record + + +def normalize_field_names(record: dict) -> dict: + for alias, name in FIELD_ALIASES.items(): + if alias in record and name not in record: + record[name] = record.pop(alias) + return record + + +def import_pyarrow(): + try: + import pyarrow + import pyarrow.parquet + except ImportError: + raise RuntimeError( + "Parquet files need the optional pyarrow dependency. Install it with: pip install 'cdx_toolkit[parquet]'" + ) from None + return pyarrow + + +def write_parquet(records: Iterable[Mapping], path: str, fields: Optional[List[str]] = None) -> int: + """Write records to a Parquet file (all columns are strings), returns the number of records written. + + Without `fields`, the columns are the fields of the first batch of records (see write_records).""" + if path == '-': + raise ValueError('Parquet output cannot be written to stdout, please set --output') + pa = import_pyarrow() + + n = 0 + schema = None + writer = None + with fsspec.open(path, 'wb') as fh: + try: + batch = [] + for record in itertools.chain(records, [None]): + if record is not None: + batch.append(dict(record)) + if len(batch) < PARQUET_BATCH_SIZE: + continue + if not batch and schema is not None: + break + if schema is None: + columns = fields or list(dict.fromkeys(k for r in batch for k in r)) + schema = pa.schema([(c, pa.string()) for c in columns]) + writer = pa.parquet.ParquetWriter(fh, schema) + table = pa.table( + {c: [None if r.get(c) is None else str(r[c]) for r in batch] for c in schema.names}, + schema=schema, + ) + writer.write_table(table) + n += len(batch) + batch = [] + finally: + if writer is not None: + writer.close() + return n + + +def read_parquet(path: str) -> Iterator[dict]: + pa = import_pyarrow() + with fsspec.open(path, 'rb') as fh: + for batch in pa.parquet.ParquetFile(fh).iter_batches(): + for record in batch.to_pylist(): + yield normalize_field_names({k: v for k, v in record.items() if v is not None}) + + +def spool_records(records: Iterable[Mapping]) -> Tuple[List[str], Iterator[dict]]: + """Write the records to a temporary file; return all their fields (in the order seen) and the records.""" + spool = tempfile.TemporaryFile('w+', encoding='utf-8') + fields = {} + for record in records: + record = dict(record) # e.g. a CaptureObject of the CDX API + fields.update(dict.fromkeys(record)) + spool.write(json.dumps(record, default=str) + '\n') + spool.seek(0) + + def read_back(): + with spool: + for line in spool: + yield json.loads(line) + + return list(fields), read_back() + + +def write_records(records: Iterable[Mapping], path: str, fmt: str, fields: Optional[List[str]] = None, + fixed_schema: bool = False) -> int: + """Write records to a file or stdout ('-'), returns the number of records written. + + The columns of CSV, TSV and Parquet are `fields`; without them, the fields of the first record if all + records have the same fields (`fixed_schema`, e.g. SQL results), otherwise all fields of all records, + found by reading the records into a temporary file first.""" + fmt = canonical_format(fmt) + if fmt in ('csv', 'tsv', 'parquet') and not fields and not fixed_schema: + fields, records = spool_records(records) + if fmt == 'parquet': + return write_parquet(records, path, fields) + n = 0 + with open_text(path, 'w') as fh: + writer = RecordWriter(fh, fmt, fields) + for record in records: + writer.write(record) + n += 1 + return n diff --git a/cdx_toolkit/repackage/__init__.py b/cdx_toolkit/repackage/__init__.py new file mode 100644 index 0000000..e531d06 --- /dev/null +++ b/cdx_toolkit/repackage/__init__.py @@ -0,0 +1,174 @@ +''' +Write captures to WARC files: copy their WARC records byte-for-byte, or replay them from a wayback machine. + +The Python equivalent of `cdxt repackage`: + + from cdx_toolkit.indexes import IndexQuery, open_index + from cdx_toolkit.repackage import repackage + + captures = open_index('cc').records(IndexQuery(urls='commoncrawl.org/*', limit=10)) + result = repackage(captures, prefix='out/COMMONCRAWL') +''' +import logging +import os +from typing import Iterable, Mapping, Optional, Sequence, Union + +from cdx_toolkit.indexes import UsageError +from cdx_toolkit.repackage.jobs import RepackageResult + + +logger = logging.getLogger(__name__) + + +def repackage( + captures: Union[str, Iterable[Mapping]], + prefix: str = 'REPACKAGE', + *, + subprefix: Optional[str] = None, + size: int = 1_000_000_000, + warc_download_prefix: str = 'https://data.commoncrawl.org', + wb: Optional[str] = None, + processes: int = 1, + parallel_readers: int = 1, + metadata_records: Sequence[str] = (), + sort: bool = True, + log_every_n: int = 1000, + creator: Optional[str] = None, + operator: Optional[str] = None, + is_part_of: Optional[str] = None, + description: Optional[str] = None, + input_format: Optional[str] = None, +) -> RepackageResult: + """Write the captures to the WARC files [-]-001.warc.gz, -002, ... (local, s3:// or hf://). + + Args: + captures: index records (e.g. IndexBackend.records()), or the path of an index file as written by + `cdxt index` (local or remote, may be a glob; '-' = stdin). + prefix, subprefix: the output files; files of this prefix and subprefix must not exist yet. + size: target byte size of each WARC file. + warc_download_prefix: where the WARC files of the captures (filename, offset, length) are read from: + https://, s3:// or hf://buckets/... + wb: replay the captures from this wayback machine instead ('ia' = the Internet Archive); for captures + without a WARC location. + processes, parallel_readers: worker processes, and concurrent readers per process. + metadata_records: files (local or remote) added as a metadata record to each WARC file. + sort: sort the captures by WARC file and offset (better read locality, keeps the captures in memory). + log_every_n: log the progress every N records (0 = off). + creator, operator, is_part_of, description: fields of the warcinfo record. + input_format: format of the index file, if `captures` is a path (default: detected). + + Returns the number of captures requested, written, and failed (could not be fetched; the others are + written). Raises UsageError for invalid options, and if the output cannot be written.""" + if wb and processes > 1: + # the minimum interval between requests to the wayback machine only holds within a process + raise UsageError('--processes cannot be used with --wb') + if isinstance(captures, str): + captures = read_index_file(captures, input_format) + check_output_is_new(prefix, subprefix) + info = warcinfo(prefix, subprefix, creator, operator, is_part_of, description) + return _run(captures, prefix, subprefix, size, warc_download_prefix, wb, processes, parallel_readers, + list(metadata_records), sort, log_every_n, info) + + +def read_index_file(path: str, input_format: Optional[str] = None) -> Iterable[dict]: + from cdx_toolkit.record_io import canonical_format, check_glob, format_from_path, import_pyarrow, read_records + + input_format = None if input_format in (None, 'auto') else canonical_format(input_format) + try: + check_glob(path) + except ValueError as e: + raise UsageError(str(e)) from None + if input_format == 'parquet' or (input_format is None and format_from_path(path) == 'parquet'): + try: + import_pyarrow() + except RuntimeError as e: + raise UsageError(str(e)) from None + return read_records(path, input_format) + + +def check_output_is_new(prefix: str, subprefix: Optional[str]) -> None: + """Every run numbers its files from the start: never overwrite (or mix with) the files of an earlier run.""" + from cdx_toolkit.repackage.writers.series import existing_warc_files + + existing = existing_warc_files(prefix, subprefix) + if existing: + raise UsageError( + f'{len(existing)} WARC file(s) with this --prefix/--subprefix exist already, e.g. {existing[0]}; ' + 'use another prefix or subprefix, or remove them' + ) + + +def warcinfo(prefix: str, subprefix: Optional[str] = None, creator: Optional[str] = None, + operator: Optional[str] = None, is_part_of: Optional[str] = None, + description: Optional[str] = None) -> dict: + """The fields of the warcinfo record at the start of each WARC file.""" + from cdx_toolkit.utils import get_version + + if not is_part_of: + is_part_of = prefix + ('-' + subprefix if subprefix else '') + info = { + 'software': 'pypi_cdx_toolkit/' + get_version(), + 'isPartOf': is_part_of, + 'description': description or 'warc extraction generated with cdx_toolkit', + 'format': 'WARC file version 1.0', # todo: if we directly read a warc, have this match the warc + } + if creator: + info['creator'] = creator + if operator: + info['operator'] = operator + return info + + +def _run(captures, prefix, subprefix, size, warc_download_prefix, wb, processes, parallel_readers, + metadata_records, sort, log_every_n, info) -> RepackageResult: + """Copy the WARC records byte-for-byte, or replay them from a wayback machine, with the async repackager + (optionally multi-process).""" + import fsspec + + from cdx_toolkit.repackage.jobs import InvalidWaybackCapture, MissingWarcLocation, RecordsSource, WaybackSource + from cdx_toolkit.repackage.urls import is_hf_url + try: + if wb: + source = WaybackSource(captures, wb) + else: + source = RecordsSource(captures, warc_download_prefix, sort=sort) + except (MissingWarcLocation, InvalidWaybackCapture) as e: + raise UsageError(str(e)) from None + + # make sure the output directory exists + prefix_fs, prefix_fs_path = fsspec.url_to_fs(prefix) + prefix_fs.makedirs(prefix_fs._parent(prefix_fs_path), exist_ok=True) + + try: + # The WARC writer supports local and S3 output; hf:// output is staged locally by the multi-process mode. + if processes > 1 or is_hf_url(prefix): + from cdx_toolkit.repackage.multiprocess import run_multiprocess_repackage + + return run_multiprocess_repackage( + source=source, + n_processes=processes, + readers_per_process=parallel_readers, + prefix_path=prefix, + writer_info=info, + writer_subprefix=subprefix, + write_paths_as_metadata_records=metadata_records, + log_every_n=log_every_n, + max_attempts=5, + uvloop=os.environ.get('CDXT_UVLOOP') == '1', + max_file_size=size, + ) + + from cdx_toolkit.repackage.repackager import Repackager + + return Repackager( + source=source, + prefix_path=prefix, + writer_info=info, + writer_subprefix=subprefix, + write_paths_as_metadata_records=metadata_records, + log_every_n=log_every_n, + n_parallel_readers=parallel_readers, + max_file_size=size, + ).run() + except (MissingWarcLocation, InvalidWaybackCapture) as e: # streamed sources: read while the jobs are run + raise UsageError(str(e)) from None diff --git a/cdx_toolkit/repackage/jobs.py b/cdx_toolkit/repackage/jobs.py new file mode 100644 index 0000000..b1f9aaa --- /dev/null +++ b/cdx_toolkit/repackage/jobs.py @@ -0,0 +1,171 @@ +''' +The captures to repackage: index records (as written by `cdxt index`) -> jobs (a WARC range, or a wayback +replay), and the result of a run. +''' +import logging +import re +from dataclasses import dataclass, field +from urllib.parse import urlparse +from typing import Iterable, Iterator, Mapping, Optional, Union + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class RepackageResult: + """Number of captures to repackage, written to WARC files, and failed (not fetched).""" + + requested: int = 0 + written: int = 0 + failed: int = 0 + + def __add__(self, other: 'RepackageResult') -> 'RepackageResult': + return RepackageResult( + self.requested + other.requested, self.written + other.written, self.failed + other.failed + ) + + +@dataclass(frozen=True) +class RangeJob: + """The bytes offset .. offset+length-1 of a WARC file (one or more WARC records).""" + + url: str + offset: int + length: int + records_count: int = 1 + # WARC filename of the record as given in the index (e.g. crawl-data/...warc.gz, relative to the download + # prefix), used to sort and shard the jobs by WARC file; `url` is authoritative for fetching. + filename: Optional[str] = None + + def __str__(self) -> str: + return f'{self.url} [{self.offset},{self.length}]' + + +@dataclass(frozen=True) +class WaybackJob: + """A capture replayed from a wayback machine (`url` is its replay URL), written as one WARC record.""" + + url: str + capture: Mapping = field(compare=False) + wb: str + records_count: int = 1 + filename: Optional[str] = None # no WARC file + + def __str__(self) -> str: + return self.url + + +Job = Union[RangeJob, WaybackJob] + + +@dataclass(frozen=True) +class RangePayload: + """The bytes read for a job (one or more gzipped WARC records).""" + + job: Job + data: bytes + + +WARC_LOCATION_FIELDS = ('filename', 'offset', 'length') + + +class MissingWarcLocation(ValueError): + """A record has no WARC location (filename, offset, length), e.g. an Internet Archive capture.""" + + +def join_warc_url(prefix: Optional[str], warc_filename: str) -> str: + """Join a download prefix with a warc_filename robustly. + + - if warc_filename is already absolute (contains '://'), return it unchanged + (supports custom queries / self-contained index files whose value is a full URL); + - if prefix is empty/None, return warc_filename unchanged; + - otherwise join with exactly one '/' (no double slash, no missing slash).""" + if '://' in warc_filename: + return warc_filename + if not prefix: + return warc_filename + return prefix.rstrip('/') + '/' + warc_filename.lstrip('/') + + +class RecordsSource: + """RangeJobs from index records (as written by `cdxt index`) with filename, offset and length fields. + + With `sort` (the default), jobs are sorted by (WARC file, record offset) for better + range-read locality. The records are then read (and validated) when the source is + created, and kept in memory; pass sort=False to stream them in input order.""" + + def __init__(self, records: Iterable[Mapping], warc_download_prefix: Optional[str], sort: bool = True): + self.warc_download_prefix = warc_download_prefix + self.jobs = (self.to_range_job(record) for record in records) + if sort: + self.jobs = sorted(self.jobs, key=lambda j: (j.filename or j.url, j.offset)) + + def iter_range_jobs(self) -> Iterator[RangeJob]: + """Yield the range jobs; `url` is authoritative for fetching, `filename` is the filename of the record.""" + yield from self.jobs + + def to_range_job(self, record: Mapping) -> RangeJob: + if not all(record.get(field) not in (None, '') for field in WARC_LOCATION_FIELDS): + raise MissingWarcLocation( + f'capture {record.get("url")} has no WARC location (filename, offset, length); ' + 'use --wb to fetch it from a wayback machine' + ) + return RangeJob( + url=join_warc_url(self.warc_download_prefix, record['filename']), + offset=int(record['offset']), + length=int(record['length']), + filename=record['filename'], + ) + + +# wayback names that map to replay URLs +WAYBACK_NAMES = {'ia': 'https://web.archive.org/web'} + + +WAYBACK_FIELDS = ('url', 'timestamp', 'status') + +# the WARC files of Common Crawl, e.g. crawl-data/CC-MAIN-2024-33/segments/..., also as a full URL +COMMON_CRAWL_WARC = re.compile(r'(^|/)crawl-data/') + + +class InvalidWaybackCapture(ValueError): + """A record cannot be replayed from a wayback machine: it lacks a field, or is a Common Crawl capture.""" + + +class WaybackSource: + """WaybackJobs from index records (url, timestamp, status), in the order of the records: the captures are + replayed from the wayback machine `wb` (a replay URL prefix, or a name of WAYBACK_NAMES).""" + + def __init__(self, records: Iterable[Mapping], wb: str): + self.wb = WAYBACK_NAMES.get(wb, wb) + self.jobs = (self.to_job(record) for record in records) + + def iter_range_jobs(self) -> Iterator[WaybackJob]: + yield from self.jobs + + def to_job(self, record: Mapping) -> WaybackJob: + from cdx_toolkit.warc import wb_replay_url + + if COMMON_CRAWL_WARC.search(str(record.get('filename') or '')): + # the old `cdxt --cc warc` refused --wb as well + raise InvalidWaybackCapture( + f'capture {record.get("url")} is a Common Crawl capture (filename {record["filename"]}): it is ' + 'copied from its WARC file, do not use --wb' + ) + missing = [name for name in WAYBACK_FIELDS if record.get(name) in (None, '')] + if missing: + raise InvalidWaybackCapture( + f'capture {record.get("url")} has no {", ".join(missing)}, which --wb needs to replay it ' + f'({", ".join(WAYBACK_FIELDS)})' + ) + url, timestamp = str(record['url']), str(record['timestamp']) + parsed = urlparse(url) + if parsed.scheme not in ('http', 'https') or not parsed.netloc: + raise InvalidWaybackCapture(f'capture url {url!r} is not an http(s) URL, which --wb needs to replay it') + if not (timestamp.isdigit() and 4 <= len(timestamp) <= 14): + raise InvalidWaybackCapture(f'capture {url} has an invalid timestamp {timestamp!r} for --wb, ' + 'e.g. 20080328121314') + # the replay compares the status as a string (Parquet and JSON input can have numbers) + capture = {**record, 'url': url, 'timestamp': timestamp, 'status': str(record['status'])} + return WaybackJob(wb_replay_url(capture, self.wb), capture, self.wb) diff --git a/cdx_toolkit/repackage/multiprocess.py b/cdx_toolkit/repackage/multiprocess.py new file mode 100644 index 0000000..5f29ff8 --- /dev/null +++ b/cdx_toolkit/repackage/multiprocess.py @@ -0,0 +1,254 @@ +"""Multi-process repackage: shard the range jobs and fetch each shard in its own process +(one asyncio event loop per CPU core), all writing into one series of output WARCs. + +A single asyncio loop saturates one CPU core on many small range-GETs, so reaching +all cores needs multiple processes. This orchestrator keeps that entirely inside +``cdxt repackage`` (no external driver): + +1. Drain the configured source once and split it into N shard files (JSONL records with the full + WARC URL), hashed by WARC filename so every record of a file stays in one shard. +2. Run one worker process per (non-empty) shard concurrently. Each worker **rotates its + output at ``--size``**; every output file is a self-contained WARC with its own + ``warcinfo`` record. +3. The file numbers come from **one counter shared by all workers**: whenever a worker + starts a new file it takes the next number, so the files are written under their final + names ``[-]-001.warc.gz``, ``-002`` … right away (and the ``warcinfo`` + record names the file it is in). Which worker writes which number depends on timing. + Output honours ``--size`` as single-process mode does (files are a target size; the last + file of each worker may be short, so up to N short files appear). +4. A failing worker (or Ctrl-C) stops all of them: the parent sets an event shared by all workers, whose + pipelines then abort the file they are writing (completed files are kept). A worker that has not + stopped after ``CANCEL_TIMEOUT`` seconds is terminated. + +The WARC writer supports local and S3 output. For other fsspec destinations (e.g. an +``hf://`` bucket) the files are staged in a local directory under their final names and +copied to the destination afterwards. +""" +import glob +import hashlib +import logging +import multiprocessing +import os +import posixpath +import shutil +import tempfile +import time + +import fsspec +from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait +from typing import Callable, Dict, List, Optional, Tuple + +from cdx_toolkit.aws import process_pool +from cdx_toolkit.record_io import RecordWriter, read_records +from cdx_toolkit.repackage.jobs import RecordsSource, RepackageResult, WaybackJob, WaybackSource +from cdx_toolkit.repackage.repackager import Repackager +from cdx_toolkit.repackage.urls import is_s3_url +from cdx_toolkit.repackage.writers.series import generate_warc_filename + +logger = logging.getLogger(__name__) + +# Concurrency for copying staged files to a non-S3 remote destination (override with CDXT_COPY_CONCURRENCY). +_COPY_CONCURRENCY = int(os.environ.get('CDXT_COPY_CONCURRENCY', '16')) + +# Seconds the workers get to stop (and abort their current file) after a failure, before they are terminated. +CANCEL_TIMEOUT = 60 + +# The shared counter of output file numbers and the cancel event, set in each worker process by _init_worker. +_FILE_SEQUENCE = None +_CANCEL = None + + +def _init_worker(file_sequence, cancel) -> None: + global _FILE_SEQUENCE, _CANCEL + _FILE_SEQUENCE = file_sequence + _CANCEL = cancel + + +def _next_file_sequence(_current: int) -> int: + """Take the next output file number from the counter shared by all workers.""" + with _FILE_SEQUENCE.get_lock(): + _FILE_SEQUENCE.value += 1 + return _FILE_SEQUENCE.value + + +def _run_worker(cfg: Dict) -> RepackageResult: + """Worker entrypoint (runs in a child process): fetch one shard into the output WARCs.""" + if cfg.get('uvloop'): + os.environ['CDXT_UVLOOP'] = '1' + # the shard is in the order of the source (sorted, unless --no-sort), streamed + records = read_records(cfg['shard'], 'jsonl') + if cfg['wb']: + source = WaybackSource(records, cfg['wb']) + else: + source = RecordsSource(records, warc_download_prefix=None, sort=False) + repackager = Repackager( + source=source, + prefix_path=cfg['prefix_path'], + writer_info=cfg['writer_info'], + writer_subprefix=cfg['subprefix'], + write_paths_as_metadata_records=cfg['write_paths_as_metadata_records'], + log_every_n=cfg['log_every_n'], + n_parallel_readers=cfg['readers'], + aws_region_name=cfg['aws_region_name'], + max_attempts=cfg['max_attempts'], + max_file_size=cfg['max_file_size'], # rotate at --size + next_file_sequence=_next_file_sequence, + cancel=_CANCEL, + ) + return repackager.run() + + +def run_worker_processes(worker: Callable, configs: List, file_sequence, mp_context, + cancel_timeout: float = None) -> List: + """Run worker(config) for each config, each in its own process; the results in the order of the configs. + + The first failure (or KeyboardInterrupt) is raised as soon as it happens, after the other workers were + stopped: the shared cancel event is set, workers that have not started are cancelled, and running workers + get `cancel_timeout` seconds to stop (abort their current file) before they are terminated. A terminated + worker cannot abort an S3 multipart upload; an S3 lifecycle rule removes incomplete uploads.""" + cancel_timeout = CANCEL_TIMEOUT if cancel_timeout is None else cancel_timeout + cancel = mp_context.Event() + ex = process_pool(len(configs), mp_context=mp_context, initializer=_init_worker, initargs=(file_sequence, cancel)) + futures = [] + try: + futures = [ex.submit(worker, config) for config in configs] + done, _ = wait(futures, return_when=FIRST_EXCEPTION) + for future in done: + future.result() # raises the first failure + results = [future.result() for future in futures] + except BaseException: + cancel.set() + for future in futures: + future.cancel() + _, running = wait(futures, timeout=cancel_timeout) + if running: + logger.error('%d worker(s) did not stop within %ds, terminating them (an S3 multipart upload of an ' + 'unfinished file may be left incomplete)', len(running), cancel_timeout) + # ProcessPoolExecutor.terminate_workers() is new in Python 3.14 + processes = list((getattr(ex, '_processes', None) or {}).values()) + for process in processes: + process.terminate() + for process in processes: + process.join(timeout=10) + ex.shutdown(wait=False, cancel_futures=True) + raise + ex.shutdown(wait=True) + return results + + +def _shard_record(job) -> dict: + """The record of a job in a shard: the WARC location with the full WARC URL as filename (read without a + download prefix), or the capture of a wayback replay.""" + if isinstance(job, WaybackJob): + return dict(job.capture) + return {'filename': job.url, 'offset': job.offset, 'length': job.length} + + +def _split_source_to_shards(source, n: int, out_dir: str) -> Tuple[List[str], List[int]]: + """Drain the source once into N shard files, hashed by WARC file (or replay URL). + + Returns the shard paths and the number of jobs in each shard.""" + paths = [os.path.join(out_dir, f'shard.{i}.jsonl') for i in range(n)] + files = [open(p, 'w', encoding='utf-8') for p in paths] + writers = [RecordWriter(f, 'jsonl') for f in files] + counts = [0] * n + total = 0 + try: + for job in source.iter_range_jobs(): + key = (job.filename or job.url or '').encode() + idx = int(hashlib.md5(key).hexdigest(), 16) % n + writers[idx].write(_shard_record(job)) + counts[idx] += 1 + total += 1 + finally: + for f in files: + f.close() + logger.info('Sharded %d range jobs into %d shards: %s', total, n, counts) + return paths, counts + + +def run_multiprocess_repackage( + *, + source, + n_processes: int, + readers_per_process: int, + prefix_path: str, + writer_info: Dict, + writer_subprefix: Optional[str], + write_paths_as_metadata_records: Optional[List[str]], + log_every_n: int, + aws_region_name: Optional[str] = None, + max_attempts: int, + uvloop: bool, + max_file_size: Optional[int] = 1_000_000_000, +) -> RepackageResult: + """Shard -> N worker processes (rotating at --size) writing one global + [-]-NNN.warc.gz series.""" + tmp_dir = tempfile.mkdtemp(prefix='cdxt_shards_') + start = time.time() + try: + shards, counts = _split_source_to_shards(source, n_processes, tmp_dir) + # empty shards would only produce WARC files without records + shards = [path for path, count in zip(shards, counts) if count > 0] or shards[:1] + + # The WARC writer supports S3 and local FS only. Other fsspec destinations + # (e.g. an hf:// bucket) are staged in a local dir and copied afterwards. + remote_non_s3 = ('://' in prefix_path) and not is_s3_url(prefix_path) + staging_dir = os.path.join(tmp_dir, 'out') + if remote_non_s3: + os.makedirs(staging_dir) + # staged under the final file names, so that the warcinfo records name the files they are in + write_prefix = os.path.join(staging_dir, posixpath.basename(prefix_path)) if remote_non_s3 else prefix_path + + configs = [ + dict( + shard=shard, + prefix_path=write_prefix, + subprefix=writer_subprefix, + writer_info=writer_info, + write_paths_as_metadata_records=write_paths_as_metadata_records, + log_every_n=log_every_n, + readers=readers_per_process, + aws_region_name=aws_region_name, + max_attempts=max_attempts, + max_file_size=max_file_size, + uvloop=uvloop, + wb=getattr(source, 'wb', None), + ) + for shard in shards + ] + + logger.info('Launching %d worker processes x %d readers', len(configs), readers_per_process) + mp_context = multiprocessing.get_context() + file_sequence = mp_context.Value('i', 0) + results = run_worker_processes(_run_worker, configs, file_sequence, mp_context) + result = sum(results, RepackageResult()) + total = result.written + n_files = file_sequence.value + fetch_elapsed = time.time() - start + logger.info('All %d workers done: %d records in %d files in %.1fs (%.0f rec/s)', + len(configs), total, n_files, fetch_elapsed, total / fetch_elapsed if fetch_elapsed else 0) + + if remote_non_s3: + _copy_staged_files(write_prefix, prefix_path) + + first_file = generate_warc_filename(prefix_path, sequence=1, writer_subprefix=writer_subprefix, gzip=True) + logger.info('Repackage complete -> %d files %s (%.1fs total)', n_files, first_file, time.time() - start) + return result + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +def _copy_staged_files(staging_prefix: str, prefix_path: str) -> List[str]: + """Copy the locally staged -... files to -... (e.g. on hf://).""" + staged = sorted(glob.glob(f'{staging_prefix}-*.warc.gz')) + + def _copy(src): + dest = prefix_path + src[len(staging_prefix):] + with open(src, 'rb') as fi, fsspec.open(dest, 'wb') as fo: + shutil.copyfileobj(fi, fo, length=8 * 1024 * 1024) + logger.info('Wrote %s', dest) + return dest + + with ThreadPoolExecutor(max_workers=max(1, min(_COPY_CONCURRENCY, len(staged)))) as ex: + return list(ex.map(_copy, staged)) diff --git a/cdx_toolkit/repackage/readers/__init__.py b/cdx_toolkit/repackage/readers/__init__.py new file mode 100644 index 0000000..ad9c90d --- /dev/null +++ b/cdx_toolkit/repackage/readers/__init__.py @@ -0,0 +1,73 @@ +''' +The readers of jobs: WARC ranges, one reader per storage (HTTP(S) in http.py, S3 in s3.py, Hugging Face buckets +in hf.py), and captures replayed from a wayback machine (wayback.py). +''' +from abc import ABC, abstractmethod +import asyncio +import contextlib +from typing import Callable + +from cdx_toolkit.repackage.jobs import Job, WaybackJob +from cdx_toolkit.repackage.urls import is_hf_url, is_s3_url + + +class RangeReader(ABC): + """Reads the bytes of a job: a range from its storage, or a wayback replay.""" + + @abstractmethod + async def read(self, job: Job) -> bytes: + """For a RangeJob exactly the bytes job.offset .. job.offset + job.length - 1 of job.url; raises (after the + reader's retries) if they cannot be read, e.g. RangeError for other bytes than requested.""" + raise NotImplementedError # pragma: no cover + + +class Readers: + """The reader of each job: the wayback reader for a WaybackJob; for a RangeJob by the URL scheme: S3 for + s3://, the HF reader for hf://, HTTP otherwise. The S3 and HF readers are created when they are first needed, + and closed with `stack`.""" + + def __init__(self, stack: contextlib.AsyncExitStack, s3_config_factory: Callable, max_attempts: int = 5, + base_backoff_seconds: float = 0.5): + from cdx_toolkit.repackage.readers.http import HttpRangeReader + from cdx_toolkit.repackage.readers.wayback import WaybackReader + + self.stack = stack + self.s3_config_factory = s3_config_factory + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + self._http = HttpRangeReader() + self._wayback = WaybackReader() + self._s3 = None + self._hf = None + self._lock = asyncio.Lock() + + async def for_job(self, job: Job) -> RangeReader: + if isinstance(job, WaybackJob): + return self._wayback + url = job.url + if is_s3_url(url): + return await self._s3_reader() + if is_hf_url(url): + return await self._hf_reader() + return self._http + + async def _s3_reader(self) -> RangeReader: + async with self._lock: + if self._s3 is None: + import aioboto3 + + from cdx_toolkit.repackage.readers.s3 import S3RangeReader + + client = aioboto3.Session().client('s3', config=self.s3_config_factory()) + client = await self.stack.enter_async_context(client) + self._s3 = S3RangeReader(client, self.max_attempts, self.base_backoff_seconds) + return self._s3 + + async def _hf_reader(self) -> RangeReader: + async with self._lock: + if self._hf is None: + from cdx_toolkit.repackage.readers import hf + + reader = hf.make_hf_reader(self.max_attempts, self.base_backoff_seconds) + self._hf = await self.stack.enter_async_context(reader) + return self._hf diff --git a/cdx_toolkit/repackage/readers/hf.py b/cdx_toolkit/repackage/readers/hf.py new file mode 100644 index 0000000..2aaeb86 --- /dev/null +++ b/cdx_toolkit/repackage/readers/hf.py @@ -0,0 +1,274 @@ +"""Hugging Face Storage Bucket support for repackage reads/writes. + +HF Storage Buckets (``hf://buckets///``) expose data only +through the Hugging Face filesystem / HTTP resolve API -- there is no S3 API yet. +So cdxt cannot reach a bucket via the aioboto3 read path. Instead WARC byte ranges +are fetched with async ``aiohttp`` ranged ``GET`` requests against the CDN: each WARC +file is resolved once (``HEAD https://huggingface.co/buckets///resolve/`` +-> redirect to a signed CDN URL, valid for about an hour), and all records of the file +are then read from the signed URL. Every ``/resolve/`` request counts against the Hub's +"Resolvers" rate limit (e.g. 5,000 per 5 minutes for free users, see +https://huggingface.co/docs/hub/en/rate-limits), the CDN requests do not. + +The records are byte-identical to the same records read from ``s3://commoncrawl``. +(Reading with ``HfFileSystem.cat_file`` instead was ~10x slower, used ~12x more memory +and dropped records under load, see the cdxt-benchmark read-backend study.) +""" +import asyncio +import collections +import contextlib +import logging +import re +import time +from typing import Tuple +from urllib.parse import parse_qs, urlparse + +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.readers import RangeReader +from cdx_toolkit.repackage.readers.http import check_range_response +from cdx_toolkit.repackage.urls import HF_SCHEME, is_hf_url + +logger = logging.getLogger(__name__) + +HF_ENDPOINT = 'https://huggingface.co' + + +def hf_fs_path(url: str) -> str: + """``hf://buckets/ns/name/key`` -> ``buckets/ns/name/key`` (HfFileSystem path).""" + if not is_hf_url(url): + raise ValueError(f'Not an hf:// URL: {url}') + return url[len(HF_SCHEME):] + + +def hf_resolve_url(url: str) -> str: + """Map an ``hf://`` bucket URL to its CDN-fronted HTTP(S) resolve URL. + + ``hf://buckets///`` + -> ``https://huggingface.co/buckets///resolve/`` + """ + path = hf_fs_path(url) + parts = path.split('/', 3) + if len(parts) < 4 or parts[0] != 'buckets': + raise ValueError( + f'Unsupported hf:// URL (expected hf://buckets///): {url}' + ) + repo = '/'.join(parts[:3]) # buckets// + key = parts[3] + return f'{HF_ENDPOINT}/{repo}/resolve/{key}' + + +def retry_after_seconds(headers, default: float) -> float: + """Seconds to wait after a 429, from the response headers. + + Prefers ``Retry-After`` (delta-seconds); else the ``t=`` field of the + standardized ``RateLimit`` header (seconds until the window resets); else + ``default``. + """ + ra = headers.get('Retry-After') + if ra: + try: + return max(0.0, float(ra)) + except (TypeError, ValueError): + pass + m = re.search(r'(?:^|[;,\s])t\s*=\s*(\d+)', headers.get('RateLimit', '') or '') + if m: + return float(m.group(1)) + return default + + +def signed_url_expiry(url: str, default_ttl: float) -> float: + """Expiry (epoch seconds) of a signed CDN URL from its ``Expires`` parameter.""" + expires = parse_qs(urlparse(url).query).get('Expires', [None])[0] + if expires and expires.isdigit(): + return float(expires) + return time.time() + default_ttl + + +class HfRangeReader(RangeReader): + """Async ranged ``GET`` requests against the HF CDN (aiohttp), resolving each WARC file once. + + The ``/resolve/`` URL of a WARC file is requested once (``HEAD``, no redirect + followed) to get the signed CDN URL it redirects to; that URL is cached (LRU) and + all records of the file are fetched from it with ``Range`` requests. The URL is + resolved again shortly before it expires, or when the CDN rejects it (401/403/410). + + Only the resolve requests count against the HF Resolvers rate limit. On ``429`` the + reader honours the server-provided wait (``Retry-After`` / ``RateLimit`` header) + instead of failing, and tallies the throttle events/wait so a run reports whether it + was rate-limited. + """ + + # Bound the wait-out so a job can ride out a 5-minute window but not hang + # forever if the limit never clears. + MAX_THROTTLE_WAIT = 360.0 # cap a single 429 backoff (seconds) + MAX_THROTTLE_RETRIES = 60 # consecutive 429s before giving up on a record + + # Resolve again this many seconds before the signed URL expires. + EXPIRY_MARGIN = 300.0 + # Lifetime assumed for a signed URL without an ``Expires`` parameter. + DEFAULT_TTL = 600.0 + # Number of signed URLs kept; the input is sorted by WARC file, so few are in use at a time. + MAX_CACHED_URLS = 4096 + + def __init__(self, session, token=None, max_attempts: int = 5, base_backoff_seconds: float = 0.5): + self.session = session + self.token = token + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + self.throttle_count = 0 # number of 429s seen + self.throttle_wait_s = 0.0 # cumulative time spent backing off 429s + self.resolve_count = 0 # number of /resolve/ requests + self.read_count = 0 # number of records read + self._policy_logged = False + # resolve URL -> (signed CDN URL, expiry); a lock per resolve URL so concurrent + # readers of the same file share one resolve request + self._signed_urls: 'collections.OrderedDict[str, Tuple[str, float]]' = collections.OrderedDict() + self._locks = {} + + def _auth_headers(self) -> dict: + return {'Authorization': f'Bearer {self.token}'} if self.token else {} + + async def _wait_for_rate_limit(self, resp, throttles: int, base_backoff_seconds: float, url: str) -> None: + self.throttle_count += 1 + default = min(30.0, base_backoff_seconds * 2 ** min(throttles, 6)) + wait = retry_after_seconds(resp.headers, default) + if not self._policy_logged: + self._policy_logged = True + logger.warning( + 'HF resolver rate limit hit (429); RateLimit=%r Policy=%r. ' + 'Throughput is now capped by the Resolvers quota.', + resp.headers.get('RateLimit'), + resp.headers.get('RateLimit-Policy'), + ) + if throttles > self.MAX_THROTTLE_RETRIES or wait > self.MAX_THROTTLE_WAIT: + raise OSError(f'HF 429: gave up after {throttles} throttle waits (last wait {wait:.0f}s) for {url}') + self.throttle_wait_s += wait + await asyncio.sleep(wait) + + async def _resolve(self, resolve: str, *, max_attempts: int, base_backoff_seconds: float) -> Tuple[str, float]: + """Resolve a file to its signed CDN URL and the URL's expiry.""" + attempt = 0 + throttles = 0 + while True: + try: + async with self.session.head(resolve, headers=self._auth_headers(), allow_redirects=False) as resp: + if resp.status == 429: + throttles += 1 + await self._wait_for_rate_limit(resp, throttles, base_backoff_seconds, resolve) + continue # does not consume the hard-error attempt budget + self.resolve_count += 1 + location = resp.headers.get('Location') + if resp.status in (301, 302, 303, 307, 308) and location: + return location, signed_url_expiry(location, self.DEFAULT_TTL) + if resp.status == 200: + # served without a redirect: read from the resolve URL itself + return resolve, time.time() + self.DEFAULT_TTL + raise OSError(f'HF HEAD {resolve} -> {resp.status}') + except Exception: # noqa: BLE001 - retry any transient resolve error + attempt += 1 + if attempt >= max_attempts: + raise + await asyncio.sleep(min(8.0, base_backoff_seconds * 2 ** (attempt - 1))) + + async def _signed_url(self, resolve: str, *, refresh: bool, **kwargs) -> str: + lock = self._locks.setdefault(resolve, asyncio.Lock()) + async with lock: + cached = self._signed_urls.get(resolve) + if cached and not refresh and cached[1] - time.time() > self.EXPIRY_MARGIN: + self._signed_urls.move_to_end(resolve) + return cached[0] + url, expiry = await self._resolve(resolve, **kwargs) + self._signed_urls[resolve] = (url, expiry) + self._signed_urls.move_to_end(resolve) + while len(self._signed_urls) > self.MAX_CACHED_URLS: + old, _ = self._signed_urls.popitem(last=False) + self._locks.pop(old, None) + return url + + async def read(self, job: RangeJob) -> bytes: + return await self.ranged_get( + job.url, job.offset, job.length, max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + ) + + async def ranged_get( + self, + url: str, + offset: int, + length: int, + *, + max_attempts: int = 5, + base_backoff_seconds: float = 0.5, + ) -> bytes: + resolve = hf_resolve_url(url) + end = offset + length - 1 # inclusive HTTP Range + attempt = 0 + throttles = 0 + refresh = False + while True: + try: + signed_url = await self._signed_url( + resolve, refresh=refresh, max_attempts=max_attempts, base_backoff_seconds=base_backoff_seconds + ) + refresh = False + headers = {'Range': f'bytes={offset}-{end}'} + if signed_url.startswith(HF_ENDPOINT): + headers.update(self._auth_headers()) + async with self.session.get(signed_url, headers=headers) as resp: + if resp.status == 429: + throttles += 1 + await self._wait_for_rate_limit(resp, throttles, base_backoff_seconds, signed_url) + continue # does not consume the hard-error attempt budget + if resp.status in (401, 403, 410): + # signed URL expired or revoked: resolve the file again + await resp.release() + refresh = True + raise OSError(f'HF CDN GET -> {resp.status}, resolving {resolve} again') + if resp.status not in (200, 206): + body = await resp.read() + raise OSError(f'HF GET {resolve} -> {resp.status}: {body[:200]!r}') + data = await resp.read() + check_range_response( + resp.status, resp.headers.get('Content-Range'), len(data), offset, length, resolve + ) + self.read_count += 1 + return data + except Exception: # noqa: BLE001 - retry any transient read error + attempt += 1 + if attempt >= max_attempts: + raise + await asyncio.sleep(min(8.0, base_backoff_seconds * 2 ** (attempt - 1))) + + async def aclose(self): + logger.info('HF reader: %d records read with %d resolve requests', self.read_count, self.resolve_count) + if self.throttle_count: + logger.warning( + 'HF resolver throttling: %d x 429, %.1fs total spent backing off ' + '(read throughput was rate-limit bound)', + self.throttle_count, + self.throttle_wait_s, + ) + + +@contextlib.asynccontextmanager +async def make_hf_reader(max_attempts: int = 5, base_backoff_seconds: float = 0.5): + """Async context manager yielding an HF reader that owns an ``aiohttp`` session. + + Imports are lazy so the optional ``hf`` extra is only required when actually + reading from a bucket. + """ + import aiohttp + + from cdx_toolkit.hf import check_huggingface_hub + + check_huggingface_hub() + from huggingface_hub import get_token + + timeout = aiohttp.ClientTimeout(total=300, sock_connect=15) + connector = aiohttp.TCPConnector(limit=0, ttl_dns_cache=300) + async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session: + reader = HfRangeReader(session, get_token(), max_attempts, base_backoff_seconds) + try: + yield reader + finally: + await reader.aclose() diff --git a/cdx_toolkit/repackage/readers/http.py b/cdx_toolkit/repackage/readers/http.py new file mode 100644 index 0000000..696faac --- /dev/null +++ b/cdx_toolkit/repackage/readers/http.py @@ -0,0 +1,84 @@ +''' +Range reads over HTTP(S). +''' +import asyncio +import re +import threading +from typing import Optional + +from cdx_toolkit.myrequests import myrequests_get +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.readers import RangeReader + + +class RangeError(OSError): + """A range request returned other bytes than requested.""" + + +def check_range_response(status: int, content_range: Optional[str], body_length: int, offset: int, length: int, + url: str) -> None: + """Raise RangeError unless an HTTP response holds exactly the bytes offset .. offset+length-1. + + A 206 response must name the requested range in Content-Range. A 200 response (the server ignored the + range) is only accepted for a range from offset 0 that is the whole file. Redirects are not followed: + a URL that redirects must be given as the redirect target.""" + end = offset + length - 1 + if status == 206: + match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+|\*)', (content_range or '').strip()) + if not match or (int(match.group(1)), int(match.group(2))) != (offset, end): + raise RangeError(f'{url}: requested bytes {offset}-{end}, got Content-Range {content_range!r}') + elif status == 200: + if offset != 0: + raise RangeError(f'{url}: the server ignored the range request for bytes {offset}-{end}') + else: + raise RangeError(f'{url}: unexpected status {status} for the range request for bytes {offset}-{end}') + if body_length != length: + raise RangeError(f'{url}: requested {length} bytes from offset {offset}, got {body_length}') + + +async def run_in_daemon_thread(func, *args, **kwargs): + """Run a blocking function in a daemon thread of its own (like asyncio.to_thread). + + Unlike asyncio.to_thread, nothing waits for a cancelled call: not the cancelled task, not asyncio.run() (which + waits for the threads of the default executor) and not the interpreter at exit.""" + loop = asyncio.get_running_loop() + future = loop.create_future() + + def set_result(result, exception): + if not future.cancelled(): + if exception is not None: + future.set_exception(exception) + else: + future.set_result(result) + + def run(): + try: + result, exception = func(*args, **kwargs), None + except BaseException as e: + result, exception = None, e + try: + loop.call_soon_threadsafe(set_result, result, exception) + except RuntimeError: + pass # the event loop is closed: nobody waits for the result anymore + + threading.Thread(target=run, daemon=True).start() + return await future + + +class HttpRangeReader(RangeReader): + """Range requests with myrequests_get (retries, and the minimum interval between requests to a host, also + across threads).""" + + async def read(self, job: RangeJob) -> bytes: + headers = {'Range': f'bytes={job.offset}-{job.offset + job.length - 1}'} + # in a thread, so that the event loop (other readers, the writer) is not blocked. If this task is + # cancelled, the thread stops at its next wait (before a retry), and nothing waits for it. + stop = threading.Event() + try: + resp = await run_in_daemon_thread(myrequests_get, job.url, headers=headers, stop=stop) + finally: + stop.set() + check_range_response( + resp.status_code, resp.headers.get('Content-Range'), len(resp.content), job.offset, job.length, job.url + ) + return resp.content diff --git a/cdx_toolkit/repackage/readers/s3.py b/cdx_toolkit/repackage/readers/s3.py new file mode 100644 index 0000000..df2f44f --- /dev/null +++ b/cdx_toolkit/repackage/readers/s3.py @@ -0,0 +1,38 @@ +''' +Range reads from S3 (aioboto3). +''' +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.readers import RangeReader +from cdx_toolkit.repackage.readers.http import RangeError +from cdx_toolkit.repackage.retries import RETRYABLE_ERRORS, with_retries +from cdx_toolkit.repackage.urls import parse_s3_uri + + +class S3RangeReader(RangeReader): + """Ranged get_object requests with an aioboto3 S3 client.""" + + def __init__(self, client, max_attempts: int, base_backoff_seconds: float): + self.client = client + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + + async def read(self, job: RangeJob) -> bytes: + bucket, key = parse_s3_uri(job.url) + end = job.offset + job.length - 1 # inclusive + + async def transfer() -> bytes: + """The whole transfer is retried: the request and reading the body (closed in any case).""" + resp = await self.client.get_object(Bucket=bucket, Key=key, Range=f'bytes={job.offset}-{end}') + async with resp['Body'] as body: + data = await body.read() + if len(data) != job.length: + raise RangeError(f'{job.url}: requested {job.length} bytes from offset {job.offset}, got {len(data)}') + return data + + return await with_retries( + transfer, + op_name=f'ranged_get {bucket}/{key}[{job.offset}:{end}]', + max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + retry_on=RETRYABLE_ERRORS + (RangeError,), + ) diff --git a/cdx_toolkit/repackage/readers/wayback.py b/cdx_toolkit/repackage/readers/wayback.py new file mode 100644 index 0000000..80cf1f1 --- /dev/null +++ b/cdx_toolkit/repackage/readers/wayback.py @@ -0,0 +1,26 @@ +''' +Captures replayed from a wayback machine (e.g. the Internet Archive), as WARC records. +''' +import logging + +from cdx_toolkit.repackage.jobs import WaybackJob +from cdx_toolkit.repackage.readers import RangeReader +from cdx_toolkit.repackage.readers.http import run_in_daemon_thread +from cdx_toolkit.repackage.warc_records import get_bytes_from_warc_record +from cdx_toolkit.warc import fetch_wb_warc + + +logger = logging.getLogger(__name__) + + +class WaybackReader(RangeReader): + """Replays a capture with fetch_wb_warc (myrequests_get: retries, and the minimum interval between requests + to the wayback machine, also across threads) and returns it as a gzipped WARC response record.""" + + async def read(self, job: WaybackJob) -> bytes: + capture = job.capture + if capture.get('mime') == 'warc/revisit': + logger.warning('revisit record being resolved for url %s %s', capture['url'], capture['timestamp']) + # in a thread, so that the event loop (other readers, the writer) is not blocked + record = await run_in_daemon_thread(fetch_wb_warc, capture, wb=job.wb) + return get_bytes_from_warc_record(record, gzip=True) diff --git a/cdx_toolkit/repackage/repackager.py b/cdx_toolkit/repackage/repackager.py new file mode 100644 index 0000000..6390e7c --- /dev/null +++ b/cdx_toolkit/repackage/repackager.py @@ -0,0 +1,468 @@ +import asyncio +import concurrent.futures +import contextlib +import logging +import os +import signal +import sys +import threading +import time +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional, Tuple + + +from cdx_toolkit.aws import botocore_config +from cdx_toolkit.repackage.jobs import RangePayload, RepackageResult +from cdx_toolkit.repackage.readers import Readers +from cdx_toolkit.repackage.readers.http import run_in_daemon_thread +from cdx_toolkit.repackage.urls import is_s3_url +from cdx_toolkit.repackage.writers.series import WarcFileSeries + + +_STOP = object() + +logger = logging.getLogger(__name__) + + +@dataclass +class ThroughputTracker: + """Items (range reads, or written records), bytes and WARC records of a reader or the writer, for the logs.""" + + start_time: float = field(default_factory=time.time) + total_bytes: int = 0 + total_requests: int = 0 + total_records: int = 0 + + def add(self, bytes_count: int = 0, records_count: int = 0, requests_count: int = 1): + self.total_bytes += bytes_count + self.total_requests += requests_count + self.total_records += records_count + + def get_stats(self) -> dict: + elapsed = time.time() - self.start_time + + return { + 'elapsed': elapsed, + 'total_bytes': self.total_bytes, + 'total_requests': self.total_requests, + 'total_records': self.total_records, + 'bytes_per_sec': self.total_bytes / elapsed if elapsed > 0 else 0, + 'mb_per_sec': (self.total_bytes / elapsed) / (1024 * 1024) if elapsed > 0 else 0, + 'requests_per_sec': self.total_requests / elapsed if elapsed > 0 else 0, + 'records_per_sec': self.total_records / elapsed if elapsed > 0 else 0, + } + + def summary(self) -> str: + stats = self.get_stats() + return ( + f"{stats['total_requests']} items, {stats['total_bytes'] / (1024 * 1024):.1f} MB " + f"in {stats['elapsed']:.1f}s, {stats['mb_per_sec']:.2f} MB/s, {stats['records_per_sec']:.2f} rec/s" + ) + + +class RepackageCancelled(Exception): + """The pipeline was stopped from outside (its `cancel` event was set), e.g. because another process failed.""" + + +class Repackager: + """Copy WARC records, byte for byte, from the WARC files of the range jobs (or the records replayed from a + wayback machine) into a series of WARC files. + + A pipeline of asyncio tasks: + - the producer drives the (sync) source in a thread and puts its range jobs into a queue, + - N reader tasks read the bytes of the jobs, with the RangeReader of each job + (HTTP, S3, Hugging Face buckets by the URL scheme, or the wayback reader; see readers/), + - one writer task writes them to the WarcFileSeries (local files or S3 objects; see writers/). + + Multi-core scaling is done with processes (see multiprocess.py), each with its own pipeline. + """ + + def __init__( + self, + prefix_path: str, + writer_info: Dict, + source, + writer_subprefix: Optional[str] = None, + write_paths_as_metadata_records: Optional[List[str]] = None, + log_every_n: int = 1000, + n_parallel_readers: int = 1, + max_attempts: int = 5, + base_backoff_seconds: float = 0.5, + range_jobs_queue_size: int = 1000, + warc_records_queue_size: int = 200, + aws_region_name: Optional[str] = None, + warc_version: str = '1.0', + min_part_size: int = 5 * 1024 * 1024, # 5 MiB (for upload) + max_file_size: Optional[int] = 1 * 1024 * 1024 * 1024, # 1 GiB (for WARC outputs) + next_file_sequence: Optional[Callable[[int], int]] = None, + cancel=None, + ): + """Set up the pipeline. + + Args: + source: The WARC ranges to repackage: an object with iter_range_jobs() (e.g. a RecordsSource). + prefix_path: Output path prefix for the WARC files. + writer_info: Dictionary containing writer metadata. + writer_subprefix: Optional subprefix for writer output paths. + write_paths_as_metadata_records: Optional list of file paths to write as metadata records. + log_every_n: Log progress every N records. + n_parallel_readers: Number of async readers per process (one writer per process). + max_attempts: Maximum retry attempts for failed operations. + base_backoff_seconds: Base backoff time in seconds for retries. + range_jobs_queue_size: Maximum size of range jobs queue. + warc_records_queue_size: Maximum size of WARC records queue. + aws_region_name: AWS region for S3 operations (default: the user's AWS configuration, else us-east-1). + warc_version: WARC format version (e.g., '1.0' or '1.1'). + min_part_size: Minimum part byte size for multipart uploads (default: 5 MiB). + max_file_size: Maximum byte size for individual WARC files (default: 1 GiB). + next_file_sequence: Number of the next output file, given the current one + (default: current + 1). + cancel: Optional event (threading.Event, multiprocessing.Event): once it is set, the pipeline stops + like on a failure (the current file is aborted) and run() raises RepackageCancelled. + """ + if n_parallel_readers < 1: + raise ValueError(f'n_parallel_readers must be at least 1, not {n_parallel_readers}') + self.source = source + self.prefix_path = prefix_path + self.writer_info = writer_info + self.writer_subprefix = writer_subprefix + self.write_paths_as_metadata_records = write_paths_as_metadata_records + self.log_every_n = log_every_n + + self.range_jobs_queue_size = range_jobs_queue_size + self.warc_records_queue_size = warc_records_queue_size + self.aws_region_name = aws_region_name + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + + # Many async readers feed a single writer. Writing is a cheap verbatim byte copy + # (+ >=5 MiB MPU parts) that one coroutine sustains easily; multi-core scaling and + # output sharding are handled by running multiple *processes* (see + # cdx_toolkit/repackage/multiprocess.py), not multiple writers on one event loop. + self.num_readers = n_parallel_readers + + self.gzip = True + + self.warc_version = warc_version + self.min_part_size = min_part_size + self.max_file_size = max_file_size + self.next_file_sequence = next_file_sequence # see WarcFileSeries + self.cancel = cancel + + def run(self) -> RepackageResult: + """Run the pipeline (runs run_async in an event loop). + + Captures that cannot be fetched are counted as failed (the others are still written); a failure to + write raises. + """ + runner = asyncio.run + if os.environ.get('CDXT_UVLOOP') == '1': + try: + import uvloop + + runner = uvloop.run + logger.info('Using uvloop event loop') + except ImportError: + logger.warning('CDXT_UVLOOP=1 but uvloop is not installed; using default asyncio loop') + try: + return runner(self._run_with_sigint()) + except asyncio.CancelledError: + if self.cancel is not None and self.cancel.is_set(): + raise RepackageCancelled('stopped (cancel event)') from None + raise + + async def _run_with_sigint(self) -> RepackageResult: + """run_async(); before Python 3.11, Ctrl-C (SIGINT) cancels it like asyncio.run() does since 3.11, so that it + stops at an await (and aborts the current file) instead of wherever KeyboardInterrupt is raised.""" + if sys.version_info >= (3, 11): + return await self.run_async() + loop = asyncio.get_running_loop() + task = asyncio.current_task() + interrupted = [] + + def on_sigint(): + interrupted.append(True) + task.cancel() + + try: + loop.add_signal_handler(signal.SIGINT, on_sigint) + except (NotImplementedError, RuntimeError, ValueError): # Windows, or not the main thread + return await self.run_async() + try: + return await self.run_async() + except asyncio.CancelledError: + if interrupted: + raise KeyboardInterrupt from None + raise + finally: + loop.remove_signal_handler(signal.SIGINT) + + async def _watch_cancel(self, pipeline: asyncio.Task) -> None: + """Cancel the pipeline once the cancel event is set.""" + while not self.cancel.is_set(): + await asyncio.sleep(0.1) + logger.warning('Stopping: cancelled from outside') + pipeline.cancel() + + def s3_read_config(self): + """botocore config of the S3 client for range reads: high throughput.""" + # region, retries and S3 settings: Common Crawl's recommended defaults, unless the + # user configured them (see cdx_toolkit.aws); an explicit aws_region_name wins + return botocore_config( + region_name=self.aws_region_name, + max_pool_connections=self.num_readers * 3, + read_timeout=300, + tcp_keepalive=True, + ) + + def s3_write_config(self): + """botocore config of the S3 client for multipart uploads (single writer).""" + return botocore_config( + region_name=self.aws_region_name, + max_pool_connections=8, + read_timeout=120, + connect_timeout=10, + ) + + async def run_async(self) -> RepackageResult: + """Run the three stages: producer of range jobs, reader tasks, writer task.""" + range_jobs_queue: asyncio.Queue = asyncio.Queue(maxsize=self.range_jobs_queue_size) + warc_records_queue: asyncio.Queue = asyncio.Queue(maxsize=self.warc_records_queue_size) + + async with contextlib.AsyncExitStack() as stack: + write_s3_client = None + if is_s3_url(self.prefix_path): + import aioboto3 + + client = aioboto3.Session().client('s3', config=self.s3_write_config()) + write_s3_client = await stack.enter_async_context(client) + + return await self._run_pipeline( + range_jobs_queue=range_jobs_queue, + warc_records_queue=warc_records_queue, + readers=Readers(stack, self.s3_read_config, self.max_attempts, self.base_backoff_seconds), + write_s3_client=write_s3_client, + ) + + async def _produce_range_jobs(self, range_jobs_queue: asyncio.Queue, stop: threading.Event) -> int: + """Drive the (sync) source in a daemon thread, feeding the async queue. + + The thread waits for queue space in short steps, so it stops soon after `stop` is set (cancellation, + or a failed writer), also when the queue is full and nobody reads from it anymore. A source that blocks + (e.g. stdin from a process that is still running) cannot be interrupted: nothing waits for its thread, + not this task when it is cancelled, not asyncio.run() and not the interpreter at exit.""" + loop = asyncio.get_running_loop() + + def drain() -> int: + count = 0 + for job in self.source.iter_range_jobs(): + if stop.is_set(): + return count + try: + future = asyncio.run_coroutine_threadsafe(range_jobs_queue.put(job), loop) + except RuntimeError: # the event loop is closed + return count + while True: + if stop.is_set(): + future.cancel() + return count + try: + future.result(timeout=0.1) + break + except concurrent.futures.TimeoutError: + pass + count += 1 + return count + + try: + count = await run_in_daemon_thread(drain) + finally: + stop.set() # the thread keeps running if this task is cancelled; let it stop + + logger.info('Generated %d range jobs', count) + return count + + async def _run_pipeline( + self, + range_jobs_queue: asyncio.Queue, + warc_records_queue: asyncio.Queue, + readers: Readers, + write_s3_client=None, + ) -> RepackageResult: + """Run the pipeline, with or without an S3 client for the output. + + A failing writer (or source) stops the whole pipeline, as does cancellation: all tasks are cancelled, + and the writer aborts the file it is writing (no incomplete WARC file or S3 object is left). + + Args: + range_jobs_queue: Queue for range jobs from the source. + warc_records_queue: Queue for WARC record payloads. + readers: The readers of the jobs (see Readers). + write_s3_client: Optional S3 client for writes S3. + """ + logger.info('Starting job generator, %d WARC readers, 1 WARC writer', self.num_readers) + stop = threading.Event() + + # Generate range jobs from the configured source (bridged sync->async in a thread). + producer = asyncio.create_task(self._produce_range_jobs(range_jobs_queue, stop)) + + # Read WARC records based on file paths and ranges + warc_readers = [ + asyncio.create_task( + self.read_warc_records( + reader_id=i, + range_jobs_queue=range_jobs_queue, + warc_records_queue=warc_records_queue, + readers=readers, + ) + ) + for i in range(self.num_readers) + ] + + # Write WARC records (a single writer owns the output shard for this process) + warc_writer = asyncio.create_task( + self.write_warc_records( + warc_records_queue=warc_records_queue, + s3_client=write_s3_client, + ) + ) + + async def feed(): + """Wait for the jobs to be produced and read, then stop the readers and the writer.""" + requested = await producer + for _ in warc_readers: + await range_jobs_queue.put(_STOP) + results = await asyncio.gather(*warc_readers) + await warc_records_queue.put(_STOP) + return requested, results + + feeder = asyncio.create_task(feed()) + tasks = [producer, *warc_readers, warc_writer, feeder] + watcher = None + if self.cancel is not None: + watcher = asyncio.create_task(self._watch_cancel(asyncio.current_task())) + tasks.append(watcher) + try: + done, _ = await asyncio.wait({feeder, warc_writer}, return_when=asyncio.FIRST_EXCEPTION) + for task in done: + task.result() # raises the exception of a failed task + requested, readers_results = await feeder + writer_tracker = await warc_writer + if watcher is not None: + watcher.cancel() + except BaseException: + stop.set() + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + raise + + reader_stats = [tracker.get_stats() for tracker, _ in readers_results] + logger.info( + 'All WARC readers completed: %d records, %.2f MB/s, %.2f rec/s', + sum(stats['total_records'] for stats in reader_stats), + sum(stats['mb_per_sec'] for stats in reader_stats), + sum(stats['records_per_sec'] for stats in reader_stats), + ) + logger.info('WARC writer completed: %s', writer_tracker.summary()) + + failed = sum(failed for _, failed in readers_results) + if failed: + logger.error('%d of %d captures could not be fetched', failed, requested) + return RepackageResult(requested=requested, written=writer_tracker.total_records, failed=failed) + + async def read_warc_records( + self, + reader_id: int, + range_jobs_queue: asyncio.Queue, + warc_records_queue: asyncio.Queue, + readers: Readers, + ) -> Tuple[ThroughputTracker, int]: + """Read WARC records based on range jobs -> enqueue RangePayload. + + Args: + reader_id: Unique identifier for this reader task. + range_jobs_queue: Queue to read the jobs (RangeJob, WaybackJob) from. + warc_records_queue: Queue to put RangePayload objects into. + readers: The readers of the jobs (see Readers). + + Returns: + The reader's throughput and the number of range jobs that could not be read. + """ + tracker = ThroughputTracker() + failed = 0 + + while True: + job = await range_jobs_queue.get() + if job is _STOP: + logger.info('WARC Reader %d stopping: %s', reader_id, tracker.summary()) + return tracker, failed + try: + reader = await readers.for_job(job) + data = await reader.read(job) + except Exception as e: + # the capture is lost, the others are still fetched; the run fails at the end + failed += 1 + logger.error('WARC Reader %d failed on %s: %r', reader_id, job, e) + continue + tracker.add(bytes_count=len(data), records_count=job.records_count) + self.log_progress(f'WARC Reader {reader_id}', tracker) + await warc_records_queue.put(RangePayload(job=job, data=data)) + + async def write_warc_records( + self, + warc_records_queue: asyncio.Queue, + s3_client=None, + ) -> ThroughputTracker: + """Write the WARC records to the series of output files (one series per process), rotating at --size. + + Args: + warc_records_queue: Queue to read RangePayload objects from. + s3_client: Optional S3 client for writing WARC files to S3. + + Returns: + The writer's throughput. + """ + series = WarcFileSeries( + self.prefix_path, + self.writer_info, + writer_subprefix=self.writer_subprefix, + metadata_paths=self.write_paths_as_metadata_records, + warc_version=self.warc_version, + gzip=self.gzip, + max_file_size=self.max_file_size, + next_file_sequence=self.next_file_sequence, + s3_client=s3_client, + min_part_size=self.min_part_size, + max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + ) + tracker = ThroughputTracker() + + # any failure is fatal, also when closing a file: the current file is aborted (removed, or its S3 upload + # aborted), the files completed before stay + try: + while True: + item = await warc_records_queue.get() + if item is _STOP: + break + assert isinstance(item, RangePayload) + await series.write(item.data) + tracker.add(bytes_count=len(item.data), records_count=item.job.records_count) + self.log_progress('WARC Writer', tracker) + await series.close() + except asyncio.CancelledError: + # stopped by the pipeline: another task (e.g. the source) failed, or the run was cancelled + logger.info('WARC writer stopped, aborting the current WARC file') + await series.abort() + raise + except BaseException: + logger.error('WARC writer failed, aborting the current WARC file') + await series.abort() + raise + return tracker + + def log_progress(self, name: str, tracker: ThroughputTracker) -> None: + """Log the progress of a reader or the writer every --log-every-n items.""" + if self.log_every_n > 0 and tracker.total_requests % self.log_every_n == 0: + logger.info('%s: %s', name, tracker.summary()) diff --git a/cdx_toolkit/repackage/retries.py b/cdx_toolkit/repackage/retries.py new file mode 100644 index 0000000..caaaf16 --- /dev/null +++ b/cdx_toolkit/repackage/retries.py @@ -0,0 +1,62 @@ +import asyncio +import logging +from typing import Tuple +from os import urandom + +try: + from botocore.exceptions import ClientError, EndpointConnectionError, HTTPClientError, IncompleteReadError +except ImportError: # pragma: no cover - S3 support is optional (cdx_toolkit[s3]) + class ClientError(Exception): + pass + + class EndpointConnectionError(Exception): + pass + + HTTPClientError = IncompleteReadError = EndpointConnectionError + +try: + from aiohttp import ClientError as AiohttpClientError # aiobotocore's transport +except ImportError: # pragma: no cover + AiohttpClientError = EndpointConnectionError + +# errors of a request or of reading its response that may be transient: timeouts, closed connections, interrupted +# or incomplete responses, S3 errors (e.g. 503 SlowDown; botocore retries those itself, too) +RETRYABLE_ERRORS = ( + TimeoutError, asyncio.TimeoutError, ConnectionError, ClientError, EndpointConnectionError, HTTPClientError, + IncompleteReadError, AiohttpClientError, +) + + +logger = logging.getLogger(__name__) + + +async def with_retries(coro_factory, *, op_name: str, max_attempts: int, base_backoff_seconds: float, + retry_on: Tuple[type, ...] = RETRYABLE_ERRORS): + """Perform operation with retries and backoff.""" + last_exc = None + for attempt in range(1, max_attempts + 1): + try: + return await coro_factory() + except retry_on as exc: + last_exc = exc + if attempt >= max_attempts: + logger.error('%s failed after %d attempts: %r', op_name, attempt, exc) + break + sleep_s = _backoff(attempt, base_backoff_seconds) + logger.warning( + '%s failed (attempt %d/%d) - retrying in %.2fs', + op_name, + attempt, + max_attempts, + sleep_s, + ) + await asyncio.sleep(sleep_s) + raise last_exc + + +def _backoff(attempt: int, base_backoff_seconds: float) -> float: + """Time to sleep based on number of attempts""" + base = base_backoff_seconds * (2 ** (attempt - 1)) + + # Add random jitter between 80-120% of base delay + return max(0.05, base * (0.8 + 0.4 * urandom(1)[0] / 255)) diff --git a/cdx_toolkit/repackage/urls.py b/cdx_toolkit/repackage/urls.py new file mode 100644 index 0000000..8a5b719 --- /dev/null +++ b/cdx_toolkit/repackage/urls.py @@ -0,0 +1,26 @@ +''' +The URL schemes of WARC files and outputs that need their own readers or writers: s3:// and hf://. +''' +from typing import Optional, Tuple + +HF_SCHEME = 'hf://' + + +def parse_s3_uri(uri: str) -> Tuple[str, str]: + """Parse a S3 URI and return bucket and prefix.""" + if not uri.startswith('s3://'): + raise ValueError(f'Not an S3 URI: {uri}') + rest = uri[5:] + i = rest.find('/') + if i <= 0 or i == len(rest) - 1: + raise ValueError(f'Malformed S3 URI: {uri}') + return rest[:i], rest[i+1:] + + +def is_s3_url(url: Optional[str]) -> bool: + return bool(url) and url.startswith('s3:/') + + +def is_hf_url(url) -> bool: + """True for an ``hf://`` URL (e.g. an hf:// WARC download prefix or job URL).""" + return bool(url) and url.startswith(HF_SCHEME) diff --git a/cdx_toolkit/repackage/warc_records.py b/cdx_toolkit/repackage/warc_records.py new file mode 100644 index 0000000..9107d95 --- /dev/null +++ b/cdx_toolkit/repackage/warc_records.py @@ -0,0 +1,71 @@ +''' +The bytes of the WARC records that cdxt repackage writes itself: warcinfo and metadata records. +''' +from io import BytesIO +import logging +import mimetypes +from pathlib import Path +from typing import Dict, Tuple, Union + +import fsspec +from warcio import WARCWriter +from warcio.recordbuilder import RecordBuilder +from warcio.recordloader import ArcWarcRecord + +logger = logging.getLogger(__name__) + + +def get_bytes_from_warc_record( + record, + warc_version: str = '1.0', + gzip: bool = False, +): + """Get byte representation of WARC record.""" + buffer = BytesIO() + warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) + warc_writer.write_record(record) + + return buffer.getvalue() + + +def get_metadata_record_from_path( + file_path: Union[str, Path], + warcinfo_id: str, +) -> ArcWarcRecord: + """Build WARC metadata record for file path. + + The `Content-Type` header is guessed. + """ + # Cast to string + file_path = str(file_path) + + with fsspec.open(file_path, 'rb') as f: + file_bytes = BytesIO(f.read()) + + warc_content_type = mimetypes.guess_type(file_path)[0] + warc_headers_dict = {} + + # Set WARC-Warcinfo-ID + warc_headers_dict['WARC-Warcinfo-ID'] = warcinfo_id + + rb = RecordBuilder() + record = rb.create_warc_record( + uri=None, + record_type='metadata', + payload=file_bytes, + http_headers=None, + warc_content_type=warc_content_type, + warc_headers_dict=warc_headers_dict, + ) + + return record + + +def warcinfo_record_bytes(warc_file_name: str, writer_info: Dict, warc_version: str, gzip: bool) -> Tuple[bytes, str]: + """The warcinfo record of a WARC file (its `filename` field names the file) and its WARC-Record-ID (the + WARC-Warcinfo-ID of the file's other records).""" + buffer = BytesIO() + warc_writer = WARCWriter(buffer, gzip=gzip, warc_version=warc_version) + warcinfo = warc_writer.create_warcinfo_record(filename=warc_file_name, info=writer_info) + warc_writer.write_record(warcinfo) + return buffer.getvalue(), warcinfo.rec_headers.get('WARC-Record-ID') diff --git a/cdx_toolkit/repackage/writers/__init__.py b/cdx_toolkit/repackage/writers/__init__.py new file mode 100644 index 0000000..6b8b235 --- /dev/null +++ b/cdx_toolkit/repackage/writers/__init__.py @@ -0,0 +1,45 @@ +''' +The output files of cdxt repackage: a local file (local.py) or an S3 object written with a multipart upload +(s3.py), and the series of WARC files they make up (series.py). +''' +from abc import ABC, abstractmethod + +from cdx_toolkit.repackage.urls import is_s3_url, parse_s3_uri + + +class OutputFile(ABC): + """One output file, written in one go: start(), write()..., then close(). + + close() completes the file; if that fails, the file aborts itself (and close() raises). abort() removes the + incomplete file (or aborts its upload), also after a failed start(); it can be called more than once.""" + + @abstractmethod + async def start(self) -> None: + raise NotImplementedError # pragma: no cover + + @abstractmethod + async def write(self, data: bytes) -> None: + raise NotImplementedError # pragma: no cover + + @abstractmethod + async def close(self) -> None: + raise NotImplementedError # pragma: no cover + + @abstractmethod + async def abort(self) -> None: + raise NotImplementedError # pragma: no cover + + +def open_output_file(path: str, s3_client=None, min_part_size: int = 5 * 1024 * 1024, max_attempts: int = 5, + base_backoff_seconds: float = 0.5) -> OutputFile: + """A new (not started) output file: an S3 object for s3://bucket/key (written with `s3_client`), else a local + file.""" + if is_s3_url(path): + from cdx_toolkit.repackage.writers.s3 import S3OutputFile + + bucket, key = parse_s3_uri(path) + return S3OutputFile(s3_client, bucket, key, min_part_size, max_attempts, base_backoff_seconds) + + from cdx_toolkit.repackage.writers.local import LocalOutputFile + + return LocalOutputFile(path) diff --git a/cdx_toolkit/repackage/writers/local.py b/cdx_toolkit/repackage/writers/local.py new file mode 100644 index 0000000..46ff19c --- /dev/null +++ b/cdx_toolkit/repackage/writers/local.py @@ -0,0 +1,60 @@ +import asyncio +import os + +import aiofiles + +from cdx_toolkit.repackage.writers import OutputFile + + +class LocalOutputFile(OutputFile): + """A local file, written with aiofiles; abort() removes it.""" + + def __init__(self, file_path: str, buffer_size: int = 8192, mode: str = 'wb'): + self.file_path = file_path + self.buffer_size = buffer_size + self.mode = mode + self.file_handle = None + self.buffer = bytearray() + + async def start(self): + opening = asyncio.ensure_future(aiofiles.open(self.file_path, self.mode)) + try: + self.file_handle = await asyncio.shield(opening) + except asyncio.CancelledError: + # the file is created anyway: finish opening it, so that abort() can remove it + self.file_handle = await opening + raise + + async def write(self, data: bytes): + self.buffer.extend(data) + if len(self.buffer) >= self.buffer_size: + await self._flush() + + async def _flush(self): + if self.buffer and self.file_handle: + await self.file_handle.write(bytes(self.buffer)) + await self.file_handle.flush() + self.buffer.clear() + + async def close(self): + """Flush and close the file; if that fails, the (incomplete) file is removed.""" + try: + if self.buffer: + await self._flush() + if self.file_handle: + file_handle, self.file_handle = self.file_handle, None + await file_handle.close() + except BaseException: + await self.abort() + raise + + async def abort(self): + """Close and remove the (incomplete) file.""" + self.buffer.clear() + if self.file_handle: + await self.file_handle.close() + self.file_handle = None + try: + os.remove(self.file_path) + except FileNotFoundError: + pass diff --git a/cdx_toolkit/repackage/writers/s3.py b/cdx_toolkit/repackage/writers/s3.py new file mode 100644 index 0000000..583a094 --- /dev/null +++ b/cdx_toolkit/repackage/writers/s3.py @@ -0,0 +1,178 @@ +import logging +from typing import List, Dict, Optional + +from cdx_toolkit.repackage.retries import with_retries +from cdx_toolkit.repackage.writers import OutputFile + +logger = logging.getLogger(__name__) + + +async def mpu_create( + s3, + bucket: str, + key: str, + *, + max_attempts: int, + base_backoff_seconds: float, +): + """Create multi part upload to S3.""" + kwargs = {'Bucket': bucket, 'Key': key} + resp = await with_retries( + lambda: s3.create_multipart_upload(**kwargs), + op_name=f'create_multipart_upload {bucket}/{key}', + max_attempts=max_attempts, + base_backoff_seconds=base_backoff_seconds, + ) + return resp['UploadId'] + + +async def mpu_upload_part( + s3, + bucket: str, + key: str, + upload_id: str, + part_number: int, + body: bytes, + max_attempts: int, + base_backoff_seconds: float, +) -> str: + """Upload a part of a multi-part upload to S3.""" + resp = await with_retries( + lambda: s3.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=part_number, + Body=body, + ), + op_name=f'upload_part {bucket}/{key}#{part_number}', + max_attempts=max_attempts, + base_backoff_seconds=base_backoff_seconds, + ) + return resp['ETag'] + + +async def mpu_complete( + s3, + bucket: str, + key: str, + upload_id: str, + parts: List[Dict], + max_attempts: int, + base_backoff_seconds: float, +): + """Send complete for multi-part upload.""" + await with_retries( + lambda: s3.complete_multipart_upload( + Bucket=bucket, Key=key, UploadId=upload_id, MultipartUpload={'Parts': parts} + ), + op_name=f'complete_multipart_upload {bucket}/{key}', + max_attempts=max_attempts, + base_backoff_seconds=base_backoff_seconds, + ) + + +async def mpu_abort(s3, bucket: str, key: str, upload_id: str): + """Abort mult-part upload.""" + try: + await s3.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id) + except Exception: + logger.exception('Failed to abort MPU %s on %s/%s', upload_id, bucket, key) + + +class S3OutputFile(OutputFile): + """An S3 object written with a multipart upload (MPU): buffers bytes, uploads >=5 MiB parts, completes on close. + + A part that cannot be uploaded (after the retries) makes the writer fail: the MPU is aborted, so no + object with missing bytes is created, and close() raises instead of completing the upload.""" + + def __init__( + self, + s3_client, + bucket: str, + key: str, + min_part_size: int, + max_attempts: int, + base_backoff_seconds: float, + ): + self.s3_client = s3_client + self.bucket = bucket + self.key = key + self.min_part_size = min_part_size + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + self.upload_id: Optional[str] = None + self.part_number = 1 + self.parts: List[Dict] = [] + self.buffer = bytearray() + self.failed = False + + async def start(self): + self.upload_id = await mpu_create( + self.s3_client, + self.bucket, + self.key, + max_attempts=self.max_attempts, + base_backoff_seconds=self.base_backoff_seconds, + ) + logger.info('Started MPU for %s (UploadId=%s)', self.key, self.upload_id) + + async def _upload_part(self, size: int): + """Upload the first `size` bytes of the buffer as the next part; they leave the buffer only once uploaded.""" + if self.failed: + raise RuntimeError(f'the upload of {self.key} failed before') + try: + etag = await mpu_upload_part( + self.s3_client, + self.bucket, + self.key, + self.upload_id, + self.part_number, + bytes(self.buffer[:size]), + self.max_attempts, + self.base_backoff_seconds, + ) + except BaseException: + await self.abort() + raise + del self.buffer[:size] + self.parts.append({'PartNumber': self.part_number, 'ETag': etag}) + self.part_number += 1 + + async def write(self, data: bytes): + self.buffer.extend(data) + while len(self.buffer) >= self.min_part_size: + await self._upload_part(self.min_part_size) + + async def close(self): + """Upload the rest and complete the MPU; if that fails, the MPU is aborted.""" + if self.failed: + raise RuntimeError(f'the upload of {self.key} failed, it was aborted') + if self.buffer: + await self._upload_part(len(self.buffer)) # aborts on failure + try: + if self.parts: + await mpu_complete( + self.s3_client, + self.bucket, + self.key, + self.upload_id, + self.parts, + self.max_attempts, + self.base_backoff_seconds, + ) + logger.info('Completed MPU for %s with %d parts.', self.key, len(self.parts)) + except BaseException: + logger.exception('Completing MPU failed for %s; attempting abort.', self.key) + await self.abort() + raise + + async def abort(self): + """Abort the MPU (no object is created); the writer cannot be used afterwards.""" + if self.failed: + return + self.failed = True + self.buffer.clear() + if self.upload_id: + logger.warning('Aborting MPU for %s', self.key) + await mpu_abort(self.s3_client, self.bucket, self.key, self.upload_id) diff --git a/cdx_toolkit/repackage/writers/series.py b/cdx_toolkit/repackage/writers/series.py new file mode 100644 index 0000000..8537307 --- /dev/null +++ b/cdx_toolkit/repackage/writers/series.py @@ -0,0 +1,159 @@ +''' +The output of a run: the series of WARC files [-]-NNN.warc.gz. +''' +import glob +import logging +import os +import posixpath +import re +from typing import Callable, Dict, List, Optional + +import fsspec + +from cdx_toolkit.repackage.urls import is_s3_url +from cdx_toolkit.repackage.warc_records import ( + get_bytes_from_warc_record, get_metadata_record_from_path, warcinfo_record_bytes, +) +from cdx_toolkit.repackage.writers import OutputFile, open_output_file + + +logger = logging.getLogger(__name__) + + +def generate_warc_filename( + dest_prefix: str, + sequence: int, + writer_subprefix: Optional[str] = None, + gzip: bool = False, +) -> str: + """The WARC file name [-]-NNN.warc[.gz]: the prefix can be a path or URL (local, s3://, + hf://), the subprefix is the user's --subprefix, the sequence is the file number (shared by all processes + of a run, at least three digits).""" + file_name = dest_prefix + '-' + if writer_subprefix is not None: + file_name += writer_subprefix + '-' + file_name += f'{sequence:03d}.warc' + if gzip: + file_name += '.gz' + + return file_name + + +def existing_warc_files(prefix_path: str, writer_subprefix: Optional[str] = None, gzip: bool = True) -> List[str]: + """The files of the series [-]-NNN.warc[.gz] (async WARC fetcher) or + [-]-NNNNNN.extracted.warc[.gz] (written by --wb in older versions) that already exist + (local, s3://, hf://, ...).""" + fs, fs_prefix = fsspec.url_to_fs(prefix_path) + name = re.escape(posixpath.basename(fs_prefix)) + '-' + if writer_subprefix is not None: + name += re.escape(writer_subprefix) + '-' + name += r'\d{3,}(\.extracted)?\.warc' + (r'\.gz' if gzip else '') + return sorted( + fs.unstrip_protocol(path) for path in fs.glob(glob.escape(fs_prefix) + '-*') + if re.fullmatch(name, posixpath.basename(path)) + ) + + +class WarcFileSeries: + """The WARC files [-]-NNN.warc.gz of a run, written one after the other. + + The first file is started with the first record, so no file is written without records. Each file starts with + its warcinfo record (and the metadata records, if any); a new file is started when the next record would make + the current one larger than `max_file_size` (each file gets at least one record). + + A failure is fatal: the file it happened in is aborted (removed, or its upload aborted), the files completed + before are kept.""" + + def __init__( + self, + prefix_path: str, + writer_info: Dict, + writer_subprefix: Optional[str] = None, + metadata_paths: Optional[List[str]] = None, + warc_version: str = '1.0', + gzip: bool = True, + max_file_size: Optional[int] = None, + next_file_sequence: Optional[Callable[[int], int]] = None, + s3_client=None, + min_part_size: int = 5 * 1024 * 1024, + max_attempts: int = 5, + base_backoff_seconds: float = 0.5, + ): + self.prefix_path = prefix_path + self.writer_info = writer_info + self.writer_subprefix = writer_subprefix + self.metadata_paths = metadata_paths or [] + self.warc_version = warc_version + self.gzip = gzip + self.max_file_size = max_file_size + # Number of the next file, given the current one (0 before the first file). The multi-process mode + # shares one counter across processes, so file names are final right away. + self.next_file_sequence = next_file_sequence or (lambda sequence: sequence + 1) + self.s3_client = s3_client + self.min_part_size = min_part_size + self.max_attempts = max_attempts + self.base_backoff_seconds = base_backoff_seconds + + self.sequence = 0 # of the current (or last) file + self._file: Optional[OutputFile] = None + self._file_size = 0 + self._records_in_file = 0 + + async def write(self, record: bytes) -> None: + """Write a (complete, possibly gzipped) WARC record, in a new file if needed.""" + if self._file is None: + await self._start_file(self.next_file_sequence(0)) + elif self._records_in_file and self.max_file_size and self._file_size + len(record) > self.max_file_size: + await self._close_file() # the finished file is kept if starting the next one fails + await self._start_file(self.next_file_sequence(self.sequence)) + logger.info(f'Rotated to new WARC file sequence {self.sequence} due to size limit') + await self._file.write(record) + self._records_in_file += 1 + self._file_size += len(record) + + async def close(self) -> None: + """Complete the current file (if it cannot be completed, it is aborted).""" + if self._file is not None: + await self._close_file() + + async def abort(self) -> None: + """Abort the current file (no-op if there is none, e.g. after a failure that aborted it already).""" + file, self._file = self._file, None + if file is not None: + await file.abort() + + async def _close_file(self) -> None: + file, self._file = self._file, None + await file.close() # aborts itself if it fails + + async def _start_file(self, sequence: int) -> None: + """Start the file with its warcinfo record (and the metadata records); if that fails, it is aborted.""" + path = generate_warc_filename(self.prefix_path, sequence, self.writer_subprefix, self.gzip) + file = open_output_file(path, self.s3_client, self.min_part_size, self.max_attempts, self.base_backoff_seconds) + # the name in the warcinfo record (local paths may have backslashes on Windows) + name = posixpath.basename(path) if is_s3_url(path) else os.path.basename(path) + logger.debug('Starting WARC file %s', name) + try: + await file.start() + header, warcinfo_id = warcinfo_record_bytes(name, self.writer_info, self.warc_version, self.gzip) + await file.write(header) + size = len(header) + if self.metadata_paths: + size += await self._write_metadata_records(file, warcinfo_id) + except BaseException: + await file.abort() + raise + self._file, self.sequence, self._file_size, self._records_in_file = file, sequence, size, 0 + + async def _write_metadata_records(self, file: OutputFile, warcinfo_id: str) -> int: + """Write a metadata record for each of the metadata paths; returns their size.""" + size = 0 + logger.info(f'Writing {len(self.metadata_paths)} metadata records to WARC ... ') + for path in self.metadata_paths: + logger.info(f'Writing metadata record from {path} ...') + record = get_metadata_record_from_path(file_path=path, warcinfo_id=warcinfo_id) + data = get_bytes_from_warc_record(record, warc_version=self.warc_version, gzip=self.gzip) + await file.write(data) + size += len(data) + logger.info(f'Metadata records added: {len(self.metadata_paths)}') + return size diff --git a/cdx_toolkit/timeutils.py b/cdx_toolkit/timeutils.py index ed4b5b9..39c5bfd 100644 --- a/cdx_toolkit/timeutils.py +++ b/cdx_toolkit/timeutils.py @@ -1,3 +1,4 @@ +import calendar import datetime from email.utils import parsedate import logging @@ -12,9 +13,6 @@ TIMESTAMP_LOW = '19780101000000' TIMESTAMP_HIGH = '29991231235959' -# if you ask for Feb we'll pad it to the 28th even if it's a leap year -days_in_month = (0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) - def pad_timestamp(ts): ''' @@ -28,11 +26,34 @@ def pad_timestamp_up(ts): ''' >>> pad_timestamp_up('199802') '19980228235959' + >>> pad_timestamp_up('200002') + '20000229235959' + >>> pad_timestamp_up('20260410') + '20260410235959' ''' - ts = ts + TIMESTAMP_HIGH[len(ts):] - month = ts[4:6] - ts = ts[:6] + str(days_in_month[int(month)]) + ts[8:] - return ts + padded = ts + TIMESTAMP_HIGH[len(ts):] + if len(ts) < 8: # no day given: the last day of the month + last_day = calendar.monthrange(int(padded[:4]), int(padded[4:6]))[1] + padded = padded[:6] + f'{last_day:02d}' + padded[8:] + return padded + + +def timestamp_range(ts): + """The first and the last second of a CDX timestamp prefix (4 to 14 digits of yyyyMMddhhmmss, padded like the + CDX API pads it) as datetimes; ValueError for an invalid one (e.g. month 13 or February 30th). + + >>> timestamp_range('202402') + (datetime.datetime(2024, 2, 1, 0, 0), datetime.datetime(2024, 2, 29, 23, 59, 59)) + """ + if not (isinstance(ts, str) and ts.isascii() and ts.isdigit() and 4 <= len(ts) <= 14): + raise ValueError(f'invalid timestamp {ts!r}, use e.g. 2008, 200803, 20080328 or 200906302359') + try: + return tuple( + datetime.datetime(*(int(padded[i:j]) for i, j in ((0, 4), (4, 6), (6, 8), (8, 10), (10, 12), (12, 14)))) + for padded in (pad_timestamp(ts), pad_timestamp_up(ts)) + ) + except ValueError: # also calendar.IllegalMonthError + raise ValueError(f'invalid timestamp {ts!r} (no such date or time)') from None def timestamp_to_time(ts): diff --git a/cdx_toolkit/utils.py b/cdx_toolkit/utils.py index aaf460c..9f6d56d 100644 --- a/cdx_toolkit/utils.py +++ b/cdx_toolkit/utils.py @@ -1,50 +1,5 @@ import cdx_toolkit -from cdx_toolkit.commoncrawl import normalize_crawl - -from argparse import Namespace -import logging - -LOGGER = logging.getLogger(__name__) def get_version(): return cdx_toolkit.__version__ - - -def setup_cdx_fetcher_and_kwargs(cmd: Namespace): - kwargs = {} - kwargs['source'] = 'cc' if cmd.crawl else cmd.cc or cmd.ia or cmd.source or None - if kwargs['source'] is None: - raise ValueError('must specify --cc, --ia, or a --source') - if cmd.wb: - kwargs['wb'] = cmd.wb - if cmd.cc_mirror: - kwargs['cc_mirror'] = cmd.cc_mirror - if cmd.crawl: - kwargs['crawl'] = normalize_crawl([cmd.crawl]) # currently a string, not a list - if getattr(cmd, 'warc_download_prefix', None) is not None: - kwargs['warc_download_prefix'] = cmd.warc_download_prefix - - cdx = cdx_toolkit.CDXFetcher(**kwargs) - - kwargs = {} - if cmd.limit: - kwargs['limit'] = cmd.limit - if 'from' in vars(cmd) and vars(cmd)['from']: # python, uh, from is a reserved word - kwargs['from_ts'] = vars(cmd)['from'] - if cmd.to: - kwargs['to'] = cmd.to - if cmd.closest: - if not cmd.get: # pragma: no cover - LOGGER.info('note: --closest works best with --get') - kwargs['closest'] = cmd.closest - if cmd.filter: - kwargs['filter'] = cmd.filter - - if cmd.cmd == 'warc' and cmd.size: - kwargs['size'] = cmd.size - - if cmd.cmd == 'size' and cmd.details: - kwargs['details'] = cmd.details - - return cdx, kwargs diff --git a/cdx_toolkit/warc.py b/cdx_toolkit/warc.py index e5e599e..4a99bc9 100644 --- a/cdx_toolkit/warc.py +++ b/cdx_toolkit/warc.py @@ -3,6 +3,7 @@ import datetime import logging import os +import warnings try: import fsspec @@ -28,7 +29,8 @@ def _is_s3_url(url): def _require_s3_deps(): if not _HAS_FSSPEC: raise RuntimeError( - 'Remote filesystem (S3) support requires optional dependencies. Install cdx_toolkit[s3].' + "Remote filesystem (S3) support requires optional dependencies. " + "Install them with: pip install 'cdx_toolkit[s3]'" ) @@ -127,6 +129,11 @@ def fake_wb_warc(url, wb_url, resp, capture): ) +def wb_replay_url(capture, wb, modifier='id_'): + """The URL of the capture in the wayback machine `wb` (with `modifier` id_: the original content).""" + return f'{wb}/{capture["timestamp"]}{modifier}/{quote(capture["url"])}' + + def fetch_wb_warc(capture, wb, modifier='id_'): for field in ('url', 'timestamp', 'status'): if field not in capture: # pragma: no cover @@ -136,9 +143,7 @@ def fetch_wb_warc(capture, wb, modifier='id_'): raise ValueError('No wayback configured') url = capture['url'] - timestamp = capture['timestamp'] - - wb_url = f'{wb}/{timestamp}{modifier}/{quote(url)}' + wb_url = wb_replay_url(capture, wb, modifier) kwargs = {} status = capture['status'] @@ -196,11 +201,17 @@ def fetch_warc_record(capture, warc_download_prefix): class CDXToolkitWARCWriter: - """Writer for WARC files. + """Writer for WARC files (deprecated: use cdx_toolkit.repackage.repackage()). The fsspec package is used for writting to local or remote file system, e.g., S3.""" def __init__(self, prefix, subprefix, info, size=1000000000, gzip=True, warc_version=None): + warnings.warn( + 'cdx_toolkit.warc.get_writer() and CDXToolkitWARCWriter are deprecated and will be removed in ' + 'a future version; use cdx_toolkit.repackage.repackage()', + FutureWarning, + stacklevel=2, + ) self.prefix = prefix self.subprefix = subprefix self.info = info diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md new file mode 100644 index 0000000..0ba4de3 --- /dev/null +++ b/docs/cli-migration-guide.md @@ -0,0 +1,324 @@ +# Migrating to the new `cdxt` command line + +The `cdxt` command line tool has a new design with two commands: + +- **`cdxt index`** selects captures from an index and writes them as JSONL, CDXJ, CSV, TSV or Parquet + (replaces `cdxt iter` and `cdxt size`). +- **`cdxt repackage`** fetches the captures listed in an index file and writes them into WARC files + (replaces `cdxt warc`, together with `cdxt index`). + +The command lines of cdx_toolkit 0.9.39 and earlier (options before the command, the URL as a +positional argument, the `iter`, `size` and `warc` commands) still work, but they are deprecated and +will be removed in a future version. They run with the new code and print a suggested new command line +on stderr: + +``` +$ cdxt --cc --limit 10 iter 'commoncrawl.org/*' +cdxt: this command line uses the old syntax, which is deprecated and will be removed in a future version; suggested replacement: cdxt index cc --url 'commoncrawl.org/*' --limit 10 --output-format csv --fields status,timestamp,url (see https://github.com/commoncrawl/cdx_toolkit#command-line-tools) +status 200, timestamp 20250804103329, url https://www.commoncrawl.org/ +... +``` + +The output of the old `iter` (text, `--jsonl`, `--csv`, the default fields `url,status,timestamp`), +`--get` and `size --details` are kept. The suggested replacement is not always exact: the new +commands have no text output (CSV is suggested instead), `--closest` always makes a single query, and +`cdxt index --count` is at most `--limit` and has no `--details`. + +The old `warc` command now runs like `cdxt index ... | cdxt repackage --no-sort`, which changes: + +- the WARC files are named `TEST-001.warc.gz`, `TEST-002.warc.gz`, ... (with `--prefix`/`--subprefix`: + `PREFIX-SUBPREFIX-001.warc.gz`) instead of `TEST-000000.extracted.warc.gz`; WARC files with the same + prefix must not exist yet +- the WARC records are copied byte-for-byte, without the added `WARC-Source-URI` and + `WARC-Source-Range` headers +- no WARC file is written without captures +- the exit status is 1 if captures could not be fetched (the others are written) + +`cdxt --cc iter --all-fields --csv` is a usage error (it failed with a traceback before). + +## Quick reference + +| Old | New | +|---|---| +| `cdxt --cc iter URL` | `cdxt index cc --url URL` | +| `cdxt --ia iter URL` | `cdxt index ia --url URL` | +| `cdxt --source CDX_API_URL iter URL` | `cdxt index CDX_API_URL --url URL` | +| `cdxt --crawl 3 iter URL` | `cdxt index cc --crawl 3 --url URL` | +| `cdxt --cc size URL` | `cdxt index cc --url URL --count` | +| `cdxt --cc warc URL` | `cdxt index cc --url URL \| cdxt repackage --prefix TEST` | +| `cdxt --ia warc URL` | `cdxt index ia --url URL \| cdxt repackage --prefix TEST --wb ia` | + +## What changed + +### All options go after the command + +Before, the index options (`--cc`, `--limit`, `--from`, ...) had to come *before* the command and the +output options *after* it. Now all options go after the command, in any order. Only `-v`/`-vv` +and `--version` can also come before the command. + +```sh +# old +cdxt --cc --limit 10 --from 2017 iter --csv 'commoncrawl.org/*' + +# new +cdxt index cc --url 'commoncrawl.org/*' --limit 10 --from 2017 --output-format csv +``` + +### The index is a positional argument + +`--cc`, `--ia` and `--source URL` are replaced by the first argument of `cdxt index`: + +| Old | New | +|---|---| +| `--cc` | `cdxt index cc` | +| `--ia` | `cdxt index ia` | +| `--source https://web.archive.org/cdx/search/cdx` | `cdxt index https://web.archive.org/cdx/search/cdx` | +| `--crawl ...` (implied `--cc`) | `cdxt index cc --crawl ...` | + +`--crawl` now has to be combined with `cc` explicitly (or a columnar index such as `cc-url-index-s3`); it +cannot be used with `ia` or other CDX API servers. + +### The URL is an option + +The URL pattern moves from a positional argument to `--url`. `--url` can be repeated, and there +are two new ways to select captures: + +| Option | Selects | Example | +|---|---|---| +| `--url PATTERN` | a URL pattern (as before) | `--url 'commoncrawl.org/*'`, `--url '*.commoncrawl.org'` | +| `--hostname HOST` | all captures of one exact hostname | `--hostname www.commoncrawl.org` | +| `--domain DOMAIN` | all captures of a domain and its subdomains | `--domain commoncrawl.org` | + +They can be repeated and combined; a capture is selected if it matches any of them: + +```sh +cdxt index cc --domain commoncrawl.org --url 'example.com/about*' --limit 100 +``` + +### Output: file or stdout, format and fields + +| Old | New | +|---|---| +| output always to stdout | stdout by default, or `-o/--output PATH` (local or remote, e.g. `s3://...`; gzipped if it ends with `.gz`) | +| `iter` (text: `status 200, timestamp ..., url ...`) | JSONL by default | +| `iter --jsonl` | default, or `--output-format jsonl` | +| `iter --csv` | `--output-format csv`, or `-o captures.csv` | +| — | `--output-format cdxj` / `tsv` / `parquet`, or `-o captures.cdxj` / `captures.tsv` / `captures.parquet` | +| `--fields url,status,timestamp` (default) | `--fields url,status,timestamp` (default: all fields) | +| `--all-fields` | default | + +The output format is detected from the `--output` file extension (`.jsonl`, `.json`, `.cdxj`, +`.cdx`, `.csv`, `.tsv`, each optionally with `.gz`, or `.parquet`, which needs `cdx_toolkit[parquet]`). For other extensions, set `--output-format`. + +The old human-readable text output is gone. The closest replacement is CSV with the old default fields: + +```sh +# old +cdxt --cc --limit 10 iter 'commoncrawl.org/*' + +# new +cdxt index cc --url 'commoncrawl.org/*' --limit 10 --fields url,status,timestamp --output-format csv +``` + +Note that the CSV columns now follow the order given in `--fields` (before, they were sorted +alphabetically) and lines end with `\n` instead of `\r\n`. + +### `size` is now `index --count` + +`--count` prints the same estimated number of captures as the old `size` command (based on the number +of index pages, so it is cheap but not exact). + +```sh +# old +cdxt --cc size 'commoncrawl.org/*' + +# new +cdxt index cc --url 'commoncrawl.org/*' --count +``` + +`index --count` has no `--details` (one line per index), and is at most `--limit`. Use `-v` to see which +indexes are queried; the deprecated `cdxt --cc size --details URL` still prints the lines. + +### `warc` is now `index` + `repackage` + +Creating WARC files is now two steps: `cdxt index` selects the captures, and `cdxt repackage` +fetches them and writes the WARC files. You can pipe one into the other, or save the captures to a +file first (handy to inspect them, or to repackage the same selection again later). + +```sh +# old +cdxt --cc --limit 10 warc --prefix COMMONCRAWL 'commoncrawl.org/*' + +# new: piped +cdxt index cc --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --prefix COMMONCRAWL + +# new: via a file +cdxt index cc --url 'commoncrawl.org/*' --limit 10 -o captures.jsonl +cdxt repackage captures.jsonl --prefix COMMONCRAWL +``` + +The `warc` options are split between the two commands: + +| Old `warc` option | New | +|---|---| +| `--url-fgrep TEXT` | `cdxt index --url-fgrep TEXT` | +| `--url-fgrepv TEXT` | `cdxt index --url-fgrepv TEXT` | +| `--prefix` (default `TEST`), `--subprefix` | `cdxt repackage --prefix` (default `REPACKAGE`), `--subprefix` | +| `--size 1000000000` | `cdxt repackage --size 1000000000`, or human readable: `--size 1GB`, `--size 100MiB` | +| `--creator`, `--operator` | `cdxt repackage --creator`, `--operator` | +| `--warc-download-prefix` | `cdxt repackage --warc-download-prefix` | +| `--wb` (global option) | `cdxt repackage --wb` | +| — | `cdxt repackage --is-part-of`, `--description` (`warcinfo` record fields) | + +**Common Crawl** captures contain their WARC location (`filename`, `offset`, `length`) and are +fetched from `--warc-download-prefix` (default `https://data.commoncrawl.org`; use +`s3://commoncrawl` in AWS us-east-1). Three things differ from the old `warc` command: + +- **File names:** `[-]-001.warc.gz`, `-002`, ... instead of + `[-]-000000.extracted.warc.gz`, `-000001`, ... +- **No empty WARC file** is written when the index selected no captures. +- **Records are copied byte-for-byte** from the source WARC files. Before, they were parsed and + re-written with added `WARC-Source-URI` and `WARC-Source-Range` headers; now the records (and + their digests) are exactly the originals. + +The records are also sorted by WARC file and offset before fetching (`--no-sort` keeps the order of +the index), and can be fetched in parallel (`--parallel-readers`, `--processes`). + +**Internet Archive** captures have no WARC location; they are replayed from the wayback machine. +Before, `--ia` did this automatically. Now pass `--wb ia` (or the URL of another wayback machine). +They are written like Common Crawl captures, to `[-]-001.warc.gz`, ... (before: +`-000000.extracted.warc.gz`, ...), and can be fetched with `--parallel-readers`: + +```sh +# old +cdxt --ia --limit 10 warc 'commoncrawl.org/*' + +# new +cdxt index ia --url 'commoncrawl.org/*' --limit 10 | cdxt repackage --wb ia + +# old: other CDX server and wayback machine +cdxt --source https://web.archive.org/cdx/search/cdx --wb https://web.archive.org/web warc 'commoncrawl.org/*' + +# new +cdxt index https://web.archive.org/cdx/search/cdx --url 'commoncrawl.org/*' \ + | cdxt repackage --wb https://web.archive.org/web +``` + +### Other option changes + +| Old | New | +|---|---| +| `--get` | removed; `cdxt index` always picks the right query type (`--closest` uses a single query) | +| `--closest TS --get` | `--closest TS` | +| `--from`, `--to`, `--filter`, `--limit` | unchanged (after the command) | +| `--cc-mirror`, `--cc-sort` | unchanged (after the command); `--cc-sort` now actually takes effect | +| `-v`, `-vv` | unchanged; before or after the command | + +`--limit` is the maximum number of captures read from the index. As with the old `warc --url-fgrep`, +the limit applies *before* `--url-fgrep`/`--url-fgrepv`, so the output can have fewer records. + +Usage errors (for example a missing `--url`, or `--crawl` with `ia`) now print a short message and +exit with code 2 instead of showing a Python traceback. + +## More examples + +```sh +# most recent 3 crawls +# old: cdxt --crawl 3 --limit 10 iter 'commoncrawl.org/*' +cdxt index cc --crawl 3 --url 'commoncrawl.org/*' --limit 10 + +# a specific crawl +# old: cdxt --crawl CC-MAIN-2024-33 iter 'commoncrawl.org/*' +cdxt index cc --crawl CC-MAIN-2024-33 --url 'commoncrawl.org/*' + +# time range at the Internet Archive +# old: cdxt --ia --limit 1 --from 2008 --to 200906302359 iter 'commoncrawl.org/*' +cdxt index ia --url 'commoncrawl.org/*' --limit 1 --from 2008 --to 200906302359 + +# filters +# old: cdxt --cc --filter '!=status:200' --filter '!=status:404' iter 'commoncrawl.org/*' +cdxt index cc --url 'commoncrawl.org/*' --filter '!=status:200' --filter '!=status:404' + +# selected fields as CSV +# old: cdxt --cc --limit 10 iter --fields=digest,length,offset --csv 'commoncrawl.org/*' +cdxt index cc --url 'commoncrawl.org/*' --limit 10 --fields digest,length,offset -o captures.csv + +# closest capture +# old: cdxt --cc --limit 3 --get --closest=20170615 iter 'commoncrawl.org/*' +cdxt index cc --url 'commoncrawl.org/*' --limit 3 --closest 20170615 + +# WARC without robots.txt, with metadata, in 100 MB files +# old: cdxt --cc --limit 100 warc --prefix CC --subprefix BLOG --size 100000000 \ +# --creator me --url-fgrepv robots.txt 'commoncrawl.org/*' +cdxt index cc --url 'commoncrawl.org/*' --limit 100 --url-fgrepv robots.txt \ + | cdxt repackage --prefix CC --subprefix BLOG --size 100MB --creator me + +# save the selection to S3, repackage it later (needs cdx_toolkit[s3]) +cdxt index cc --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o s3://my-bucket/captures.jsonl.gz +cdxt repackage s3://my-bucket/captures.jsonl.gz --prefix s3://my-bucket/warcs/COMMONCRAWL +``` + +## New: the columnar URL index + +Besides the CDX API, `cdxt index` can query Common Crawl's columnar URL index (parquet) with DuckDB +or AWS Athena. This was not possible with the old commands: + +```sh +# DuckDB over s3://commoncrawl/cc-index/table/cc-main/warc/ (needs AWS credentials and cdx_toolkit[duckdb]) +cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.jsonl + +# your own SQL query (examples/homepages.sql in the cdx_toolkit repository) +cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o homepages.csv + +# AWS Athena (needs cdx_toolkit[athena]) +cdxt index 'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://my-bucket/athena/' \ + --hostname commoncrawl.org --crawl CC-MAIN-2024-33 -o captures.jsonl +``` + +See the README for all index types. + +## Users of the `feat/warc-range-sources` branch + +The `repackage` command of the unreleased `feat/warc-range-sources` branch (#73) is split into +`cdxt index` (selecting the records) and `cdxt repackage` (fetching them): + +| Branch | New | +|---|---| +| `cdxt --crawl C repackage --target-source sql --engine athena --athena-database D --athena-s3-output S3 --hostnames H ...` | `cdxt index 'athena://athena.us-east-1.amazonaws.com/D?S3OutputLocation=S3' --crawl C --hostname H -o captures.csv` + `cdxt repackage captures.csv ...` | +| `... --engine duckdb --duckdb-index-path PATH` | `cdxt index PATH --index-backend duckdb ...` (or `cc-url-index-s3`, `cc-url-index-hf`) | +| `--hostnames H1 H2`, `--domains D1 D2` | `--hostname H1 --hostname H2`, `--domain D1 --domain D2` | +| `--query SQL`, `--query-file FILE` | `cdxt index ... --sql FILE` (`--sql -` reads stdin) | +| — | new for SQL indexes: `--from`/`--to` (`fetch_time`), `--language` (primary content language, ISO 639-3), Parquet output (`-o captures.parquet`, `cdx_toolkit[parquet]`) | +| `--confirm-cost` | `cdxt index --yes` | +| `--target-source cdx --cdx-path PATH [--cdx-glob GLOB]` | `cdxt repackage 'PATH/GLOB'` (CDX files are read directly) | +| `--target-source csv --csv-path FILE` | `cdxt repackage FILE` | +| `--range-jobs-output FILE --no-fetch` | `cdxt index ... -o FILE` | +| `--csv-self-contained` (`warc_url` column) | removed; `warc_filename` (or `filename`) may hold a full URL (`s3://`, `https://`, `hf://`), which is used as it is, otherwise it is joined with `--warc-download-prefix` | +| `--limit N` (global option) | `cdxt index --limit N` (repackage does not filter) | +| `--parallel N`, `--parallel_readers N` | `cdxt repackage --parallel-readers N` | +| `--no-sort-ranges` | `cdxt repackage --no-sort` | +| `--write-paths-as-metadata-records F1 F2` | `cdxt repackage --add-metadata-record F1 --add-metadata-record F2` | +| `--log_every_n N` | `cdxt repackage --log-every-n N` | +| `--keep-shards` | removed; with `--processes`, all processes write directly into one `-001`, `-002`, ... series (`--subprefix` now works too) | +| `--hf-reader fsspec\|cdn` | removed; `hf://` WARCs are always read from the CDN, with one rate-limited `/resolve/` request per WARC file instead of per record (`fsspec` was ~10x slower and dropped records under load in the benchmarks) | +| `--processes`, `--warc-download-prefix`, `--prefix`, `--subprefix`, `--size`, `--creator`, `--operator`, `--is-part-of`, `--description` | unchanged (`--size` also accepts e.g. `1GB`) | +| `cdxt filter_cdx INPUT_BASE FILTER_FILE OUTPUT_BASE --input-glob GLOB --parallel N` | `cdxt index 'INPUT_BASE/GLOB' --url-file FILTER_FILE -o OUTPUT_BASE --processes N` (one output per CDX file). Add `--url-file-type domain` to match the entries like `--filter-type url` did (`example.com` = the domain with subdomains, entries with a path = URL prefixes); without it, the entries are URL patterns like `--url` | +| `--filter-type surt`, `--overwrite`, `--limit` (per file) | removed; the output directory has to be new or empty, `--limit` only works with a single output | + +## Python API + +`cdx_toolkit.CDXFetcher`, `cdx.iter()`, `cdx.get_size_estimate()` and `capture.content` have not +changed. New: the Python equivalents of the new commands, which run the same code as the command line +(see the README): + +```python +from cdx_toolkit.indexes import IndexQuery, open_index +from cdx_toolkit.repackage import repackage + +index = open_index('cc') # cdxt index cc +query = IndexQuery(urls='commoncrawl.org/*', limit=10) # --url 'commoncrawl.org/*' --limit 10 +repackage(index.records(query), prefix='COMMONCRAWL') # | cdxt repackage --prefix COMMONCRAWL +``` + +`cdx_toolkit.warc.get_writer()` (`CDXToolkitWARCWriter`) is deprecated and will be removed in +a future version; `repackage()` replaces it. diff --git a/docs/settings.md b/docs/settings.md new file mode 100644 index 0000000..ab3b4e4 --- /dev/null +++ b/docs/settings.md @@ -0,0 +1,67 @@ +# Settings + +## HTTP requests: rate limits and retries + +Requests over HTTP(S) go through `cdx_toolkit/myrequests.py`: + +- CDX API queries: `index.commoncrawl.org` for `cc` (and its list of crawls, `collinfo.json`, which is + cached for a day in `CDXT_CACHE_DIR`, default `~/.cache/cdx_toolkit/`), `web.archive.org/cdx/search/cdx` + for `ia`, and other CDX API servers; +- WARC records read over HTTP(S), e.g. from `https://data.commoncrawl.org` (`cdxt repackage` with the + default `--warc-download-prefix`, and `capture.content` in Python); +- wayback replays, e.g. from `https://web.archive.org/web` (`cdxt repackage --wb ia`). + +Reads from `s3://` (see [S3 settings](#s3-settings)), `hf://`, and the queries of DuckDB and Athena do not use it. + +### Minimum interval between requests + +Requests to the same host start at least a minimum interval apart, also when they are made from several +threads (e.g. `cdxt repackage --parallel-readers`). Every attempt counts, also a retry: it waits for its +backoff, then for its turn. After a request, including its retries, the next one waits the interval again. + +| Host | Interval | Environment variable | +|---|---|---| +| `index.commoncrawl.org` | 1.0 s | `CDXT_CC_INDEX_MIN_RETRY_INTERVAL` | +| `data.commoncrawl.org` | 0.55 s | `CDXT_CC_DATA_MIN_RETRY_INTERVAL` | +| `web.archive.org` (CDX API and replays) | 6.0 s | `CDXT_IA_MIN_RETRY_INTERVAL` | +| any other host | 3.0 s | `CDXT_DEFAULT_MIN_RETRY_INTERVAL` | + +The interval holds within a process: `cdxt repackage --processes N` makes up to N times as many requests +to a host. So reads from `https://data.commoncrawl.org` are limited to at most about 1.8 requests per second +per process, whatever `--parallel-readers` is; for faster reads, use `s3://commoncrawl` in AWS us-east-1. +`--wb` cannot be used with `--processes` for this reason. Please do not lower the intervals for Common +Crawl's and the Internet Archive's servers: they are shared by everybody. + +### Retries + +| Response | What happens | +|---|---| +| status 429, 500, 502, 503, 504, 509 (slow down, temporary errors) | retried without a limit; the wait starts at twice the host's interval and doubles up to 60 s; logged as a warning from the 6th retry | +| connection error, timeout (30 s to connect, 30 s between bytes), broken response | retried after 60 s; a warning from the 11th error (`CDXT_WARNING_AFTER_N_ERRORS`, default 10), fails after 101 errors (`CDXT_MAX_ERRORS`, default 100) | +| unknown hostname (DNS error) | fails at once, unless the host answered before in this process (or is `data.commoncrawl.org`, `commoncrawl.s3.amazonaws.com` or `web.archive.org`): then it is retried like a connection error | +| status 400 or 404 from a CDX API | not an error: no captures, or past the last page | +| status 404 when replaying a capture with status 404, or a revisit (`-`) | not an error: the replay is the 404 | +| other status 400 or 404, and other 4xx/5xx statuses | fails at once, no retry | + +Redirects are not followed. Requests send the `User-Agent` `pypi_cdx_toolkit/`. + +In `cdxt repackage`, a capture whose request fails is counted as failed (the other captures are still +written, see the README). When the run is stopped, the waits end at once; a request in progress ends with +its timeout. + +## S3 settings + +For requests to S3 (e.g. `s3://commoncrawl`), `cdxt` uses Common Crawl's +recommended AWS settings: region `us-east-1`, `retry_mode = adaptive`, +`max_attempts = 100000`, and for S3 `addressing_style = virtual`, +`payload_signing_enabled = false` and `max_concurrent_requests = 2`. +Settings you configured yourself win: environment variables (`AWS_REGION`, +`AWS_DEFAULT_REGION`, `AWS_RETRY_MODE`, `AWS_MAX_ATTEMPTS`), the AWS config file +of your profile (`~/.aws/config`, `AWS_PROFILE`), and options in the fsspec +configuration. `CDXT_AWS_DEFAULTS=0` turns the defaults off. + +The Athena client uses the same retry settings (its region is part of the +`athena://` location). DuckDB has no adaptive retry mode and no limit for +concurrent requests; for S3 it uses the region (`s3_region`), the addressing +style (`s3_url_style`) and `max_attempts` (`http_retries`, with 1 second between +retries). diff --git a/docs/usage/aws-athena-s3.md b/docs/usage/aws-athena-s3.md new file mode 100644 index 0000000..e6c9d62 --- /dev/null +++ b/docs/usage/aws-athena-s3.md @@ -0,0 +1,117 @@ +# AWS: Athena and S3 + +Common Crawl's data is in the S3 bucket `s3://commoncrawl` in AWS `us-east-1`: the WARC files, the CDX +files and the [columnar URL index](https://commoncrawl.org/columnar-index). With an AWS account, you can +query the columnar index with [Athena](https://aws.amazon.com/athena/) (serverless SQL, billed per TB +scanned) or with DuckDB, and copy the selected records from S3. This is the fastest setup for large +selections, especially when `cdxt repackage` runs in `us-east-1` (e.g. on EC2), where reads from +`s3://commoncrawl` are fast and free of transfer costs. + +You need: + +- `pip install 'cdx_toolkit[athena,s3]'` (and `[duckdb]` for DuckDB); +- AWS credentials, found like boto3 finds them (environment, `~/.aws` profiles, SSO, instance roles). + Reads from `s3://commoncrawl` need credentials of any AWS account; anonymous requests are refused; +- for Athena: an S3 bucket of your own for the query results (`S3OutputLocation`), and the table of the + columnar index in your account: run the `CREATE EXTERNAL TABLE` and `MSCK REPAIR TABLE` statements of + [cc-index-table](https://github.com/commoncrawl/cc-index-table#query-the-table-in-amazon-athena) once + (database `ccindex`, table `ccindex`), and repeat `MSCK REPAIR TABLE` to add new crawls. + +The region, retries and timeouts of the S3, Athena and DuckDB requests are described in +[settings.md](../settings.md). + +### Query URL Index via Athena + +The Athena INDEX names the region, the database and the result location: + +```sh +ATHENA='athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://my-bucket/athena-results/' +``` + +Add `&table=...` if your table is not called `ccindex`. + +Send a query for a specific host name and a few crawls: + +```sh +cdxt index "$ATHENA" --hostname commoncrawl.org --crawl CC-MAIN-2026-17,CC-MAIN-2026-13 -o captures.csv +``` + +`--dry-run` prints the SQL that `cdxt` generates, and `--count` runs a `count(*)` instead. The query is +restricted to the partitions of the given crawls and to `subset = 'warc'`; `--domain` and `--hostname` use +the column `url_host_name_reversed`, which follows the sort order of the index, so Athena skips most of +the data. With `-v`, `cdxt` logs the amount of data each query scanned and its estimated cost. + +Get all records of a TLD for all crawls with your own SQL (`va.sql`): + +```sql +SELECT url, fetch_time, fetch_status, content_mime_type, crawl, + warc_filename, warc_record_offset, warc_record_length +FROM ccindex +WHERE subset = 'warc' + AND url_host_tld = 'va' +``` + +```sh +cdxt index "$ATHENA" --sql va.sql -o va.csv +``` + +`--sql` runs the query as it is. Without a `crawl` condition, it scans the `url_host_tld` column of +every crawl, so `cdxt` asks for confirmation first (`--yes` skips the question). Try it on one crawl +first (`AND crawl = 'CC-MAIN-2026-17'`) to see the amount of data scanned. For `cdxt repackage`, a query +has to select `url`, `warc_filename`, `warc_record_offset` and `warc_record_length`. + +Get the records of a list of hosts with a table join (like cc-index-table's +[count-domains-alexa-top-1m.sql](https://github.com/commoncrawl/cc-index-table/blob/main/src/sql/examples/cc-index/count-domains-alexa-top-1m.sql)). +Athena joins tables only, so upload the list to S3 and create a table for it once: + +```sh +aws s3 cp hosts.csv s3://my-bucket/hosts/hosts.csv # one host per line, e.g. www.commoncrawl.org + +cat > create-hosts.sql <<'EOF' +CREATE EXTERNAL TABLE IF NOT EXISTS hosts (host string) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION 's3://my-bucket/hosts/' +EOF +aws athena start-query-execution --query-string file://create-hosts.sql \ + --query-execution-context Database=ccindex \ + --result-configuration OutputLocation=s3://my-bucket/athena-results/ +``` + +(or run the statement in the Athena console). Then join it with the index (`hosts.sql`): + +```sql +SELECT cc.url, cc.fetch_time, cc.fetch_status, cc.crawl, + cc.warc_filename, cc.warc_record_offset, cc.warc_record_length +FROM ccindex AS cc +JOIN hosts AS h ON cc.url_host_name = h.host +WHERE cc.crawl = 'CC-MAIN-2026-17' + AND cc.subset = 'warc' +``` + +```sh +cdxt index "$ATHENA" --sql hosts.sql --yes -o hosts-captures.csv +``` + +### Repackage via S3 Bucket + +Read the WARC records from `s3://commoncrawl` and write the WARC files to your bucket: + +```sh +cdxt repackage hosts-captures.csv --warc-download-prefix s3://commoncrawl \ + --prefix s3://my-bucket/warc/HOSTS --size 1GB --processes 8 --parallel-readers 16 +``` + +- `--warc-download-prefix s3://commoncrawl` reads with your credentials; in `us-east-1`, it is much + faster than the default `https://data.commoncrawl.org`, which has a per-process rate limit. +- `--processes` (e.g. the number of vCPUs) and `--parallel-readers` (concurrent reads per process) set + the throughput. The files are named `HOSTS-001.warc.gz`, `-002`, … whatever the number of processes. +- `--prefix` can also be a local path; `--add-metadata-record FILE` adds a file (e.g. the query) as a + metadata record to each WARC file. + +`cdxt index` can write the captures to S3 as well (`-o s3://my-bucket/hosts-captures.csv`), and +`cdxt repackage` can read them from there. + +DuckDB works on the same data without Athena: `cdxt index cc-url-index-s3` queries +`s3://commoncrawl/cc-index/table/cc-main/warc/` with your credentials, with the same options, e.g. +`cdxt index cc-url-index-s3 --domain commoncrawl.org --crawl CC-MAIN-2026-17 -o captures.csv`. It costs +nothing but reads the Parquet footers of every file of the crawls, so it is fastest in `us-east-1`. diff --git a/docs/usage/http.md b/docs/usage/http.md new file mode 100644 index 0000000..b017784 --- /dev/null +++ b/docs/usage/http.md @@ -0,0 +1,141 @@ +# HTTP + +Everything in this guide works without an account or credentials: the indexes and the WARC files are +read over HTTPS from Common Crawl's and the Internet Archive's public servers. + +- Common Crawl's CDX API (`index.commoncrawl.org`, one index per crawl) is described in + [CDXJ index](https://commoncrawl.org/cdxj-index); the WARC files are on `https://data.commoncrawl.org`. +- The Internet Archive's CDX API and replays are described in the + [Wayback Machine APIs](https://archive.org/help/wayback_api.php). + +These servers are shared by everybody, so `cdxt` keeps a minimum interval between requests to each of +them (e.g. 1 s for `index.commoncrawl.org`, 6 s for `web.archive.org`) and retries when they ask it to +slow down; see [settings.md](../settings.md). That makes HTTP the right choice for small selections: +some hosts, URL prefixes or a few crawls. For large selections (a TLD, a long list of hosts), use the +columnar index (below) or, faster, [AWS](aws-athena-s3.md) or [Hugging Face](huggingface.md). + +```sh +pip install cdx_toolkit # CDX API and HTTP downloads +pip install 'cdx_toolkit[duckdb]' # also the columnar index with DuckDB +``` + +## Small scale + +### Query CDX API Server + +All captures of a host in the two most recent crawls (`--crawl 2`; or name them, e.g. +`--crawl CC-MAIN-2026-17,CC-MAIN-2026-13`): + +```sh +cdxt index cc --hostname commoncrawl.org --crawl 2 -o captures.jsonl +``` + +Check the size of a selection first: `--count` is a cheap estimate based on the number of index pages, +and `--dry-run` prints the API requests without sending them: + +```sh +cdxt index cc --hostname commoncrawl.org --crawl 2 --count +cdxt index cc --hostname commoncrawl.org --crawl 2 --dry-run +``` + +Only successful HTML captures, and only some fields (`--filter` uses the CDX API filter syntax, see the +README): + +```sh +cdxt index cc --domain commoncrawl.org --crawl 2 \ + --filter =status:200 --filter '~mime:text/html' \ + --fields url,timestamp,status,mime -o captures.csv +``` + +`--url` takes the CDX API's URL patterns, e.g. `'commoncrawl.org/blog/*'` for a prefix or +`'*.commoncrawl.org'` for a domain. + +The Internet Archive works the same way, with `--from`/`--to` instead of crawls: + +```sh +cdxt index ia --url 'commoncrawl.org/*' --from 2012 --to 2013 --limit 100 -o ia-captures.jsonl +``` + +Write the captures to WARC files. Common Crawl captures are copied byte-for-byte from +`https://data.commoncrawl.org`; Internet Archive captures have no WARC location and are replayed from the +Wayback Machine with `--wb ia`: + +```sh +cdxt repackage captures.jsonl --prefix out/COMMONCRAWL +cdxt repackage ia-captures.jsonl --wb ia --prefix out/IA +``` + +Both commands can also be piped: + +```sh +cdxt index cc --hostname commoncrawl.org --crawl 1 --limit 20 | cdxt repackage --prefix out/CC-SAMPLE +``` + +## Large scale + +### Query URL Index via DuckDB + +The [columnar URL index](https://commoncrawl.org/columnar-index) has one row per capture with columns +such as `url_host_tld`, `url_host_registered_domain`, `content_languages` and the WARC location, in +Parquet files partitioned by `crawl` and `subset` (`warc`, `robotstxt`, `crawldiagnostics`). DuckDB +reads only the columns and row groups a query needs. + +`cdxt index cc-url-index-s3` reads it from `s3://commoncrawl`, which needs AWS credentials (see the +[AWS guide](aws-athena-s3.md)). Without credentials, DuckDB reads the same files over HTTPS. HTTPS can't +list a directory, so the query names the files; every crawl has a list of them in +`crawl-data//cc-index-table.paths.gz`: + +```sh +CRAWL=CC-MAIN-2026-17 +curl -s https://data.commoncrawl.org/crawl-data/$CRAWL/cc-index-table.paths.gz | gunzip \ + | grep '/subset=warc/' | sed "s|^|'https://data.commoncrawl.org/|; s|\$|'|" | paste -sd, - > files.txt +``` + +All captures of a TLD (here `.va`) in that crawl, in `tld.sql`: + +```sh +cat > tld.sql < hosts.sql <-001.warc.gz`, `-002`, …, each about `--size` (default 1 GB), with a +`warcinfo` record that lists the command line. diff --git a/docs/usage/huggingface.md b/docs/usage/huggingface.md new file mode 100644 index 0000000..f410a0b --- /dev/null +++ b/docs/usage/huggingface.md @@ -0,0 +1,100 @@ +# Hugging Face + +Recent Common Crawl crawls are also on Hugging Face, in the Storage Bucket +`hf://buckets/commoncrawl/commoncrawl` with the same paths as `s3://commoncrawl`: the WARC files and the +columnar URL index (`cc-index/table/cc-main/warc/`). See +[Getting started with Common Crawl data on Hugging Face](https://commoncrawl.org/blog/getting-started-with-common-crawl-data-on-hugging-face). +`cdxt` can query the index there with DuckDB, read the WARC records from the bucket, write the WARC +files to a bucket of your own, and run as a [Hugging Face Job](https://huggingface.co/docs/huggingface_hub/en/guides/jobs), +next to the data. + +```sh +pip install 'cdx_toolkit[hf,duckdb]' +hf auth login # or set HF_TOKEN +``` + +Reads from the public bucket work without a token, but with lower +[rate limits](https://huggingface.co/docs/hub/en/rate-limits); writing to your bucket needs a token with +write access to it. + +## Query URL index on Hugging Face via DuckDB + +`cc-url-index-hf` takes the same options as `cc-url-index-s3`: + +```sh +cdxt index cc-url-index-hf --hostname commoncrawl.org --crawl CC-MAIN-2026-17 -o captures.jsonl +``` + +This is slow: DuckDB reads the Parquet files through the Hub API, and each of a crawl's 300 files takes +several requests whose latency (and rate limits) add up to many minutes per crawl. Restrict the query +to as few crawls as possible, try it on a single file with `--sql` first: + +```sql +SELECT url, fetch_time, fetch_status, + warc_filename, warc_record_offset, warc_record_length +FROM read_parquet('hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-17/subset=warc/part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet') +WHERE url_host_tld = 'va' +``` + +```sh +cdxt index cc-url-index-hf --sql va.sql --yes -o va.csv +``` + +or run the query as an HF Job (below). For large queries, S3 is faster (see the +[AWS guide](aws-athena-s3.md)). + +## Repackage via HF Bucket + +```sh +cdxt repackage captures.jsonl --warc-download-prefix hf://buckets/commoncrawl/commoncrawl \ + --prefix hf://buckets///warc/CC --parallel-readers 8 +``` + +`cdxt repackage` resolves each WARC file once and then reads its records from the signed download URL, +so the rate-limited Hub API sees one request per WARC file, not per record. `--prefix` can also be a +local path, and `cdxt index` can write to a bucket (`-o hf://buckets///captures.jsonl`). + +## HF Jobs + +An HF Job runs a command on Hugging Face's infrastructure, close to the bucket, without a machine of +your own (it needs a Hugging Face account with Jobs enabled, e.g. PRO or an organization with credits). +`hf jobs uv run` runs a Python script; this one (`cdxt_job.py`) installs `cdx_toolkit` and runs `cdxt` +with the arguments of the job: + +```python +# /// script +# requires-python = ">=3.10" +# dependencies = ["cdx_toolkit[hf,duckdb]"] +# /// +import sys + +from cdx_toolkit.cli import main + +main(args=sys.argv[1:]) +``` + +Two jobs: the first selects the captures and writes them to your bucket, the second writes them to WARC +files in your bucket. `-s HF_TOKEN` passes your token (from the environment or `hf auth login`) to the +job, so it can write to the bucket. + +1) `cdxt index`, writing to an HF bucket: + +```sh +hf jobs uv run -s HF_TOKEN --timeout 2h cdxt_job.py \ + index cc-url-index-hf --hostname commoncrawl.org --crawl CC-MAIN-2026-17 \ + -o hf://buckets///captures.jsonl +``` + +2) `cdxt repackage`: + +```sh +hf jobs uv run -s HF_TOKEN --flavor cpu-upgrade --timeout 2h cdxt_job.py \ + repackage hf://buckets///captures.jsonl \ + --warc-download-prefix hf://buckets/commoncrawl/commoncrawl \ + --prefix hf://buckets///warc/CC --parallel-readers 8 +``` + +`hf jobs uv run` prints the job's logs until it finishes; with `--detach`, it prints the job ID and +returns, and `hf jobs logs ` and `hf jobs ps` follow it. `--timeout` sets the maximum duration, +`--flavor` the hardware (`hf jobs hardware` lists them), and `--namespace` runs the job in an +organization. diff --git a/examples/homepages.sql b/examples/homepages.sql new file mode 100644 index 0000000..b8c95fb --- /dev/null +++ b/examples/homepages.sql @@ -0,0 +1,15 @@ +-- The homepages of the .va domain in one crawl: successful captures of the path "/" without a query string. +-- +-- cdxt index cc-url-index-s3 --sql examples/homepages.sql --yes -o homepages.csv +-- cdxt repackage homepages.csv --prefix HOMEPAGES +-- +-- --sql runs the query as it is, so it names the Parquet files it reads (here one crawl of the columnar URL +-- index on S3, which needs AWS credentials); INDEX (cc-url-index-s3) only selects DuckDB. cdxt repackage needs +-- the columns url, warc_filename, warc_record_offset and warc_record_length. +SELECT url, fetch_time, fetch_status, content_mime_type, + warc_filename, warc_record_offset, warc_record_length +FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-33/subset=warc/*.parquet') +WHERE url_host_tld = 'va' + AND url_path = '/' + AND url_query IS NULL + AND fetch_status = 200 diff --git a/examples/iter-and-warc.py b/examples/iter-and-warc.py index 1d1b820..75e66fc 100755 --- a/examples/iter-and-warc.py +++ b/examples/iter-and-warc.py @@ -1,33 +1,24 @@ #!/usr/bin/env python +''' +Select captures from Common Crawl's CDX index and write them into a WARC file, +like `cdxt index cc --url 'commoncrawl.org/blog/*' --filter =status:200 --limit 10 | cdxt repackage ...` +''' +from cdx_toolkit.indexes import IndexQuery, open_index +from cdx_toolkit.repackage import repackage -import cdx_toolkit +index = open_index('cc') +query = IndexQuery(urls='commoncrawl.org/blog/*', filters='=status:200', limit=10) -cdx = cdx_toolkit.CDXFetcher(source='cc') -url = 'commoncrawl.org/blog/*' +captures = [] +for capture in index.records(query): + print('extracting url', capture['url'], 'timestamp', capture['timestamp']) + captures.append(capture) -warcinfo = { - 'software': 'pypi_cdx_toolkit iter-and-warc example', - 'isPartOf': 'EXAMPLE-COMMONCRAWL', - 'description': 'warc extraction', - 'format': 'WARC file version 1.0', -} - -with cdx_toolkit.warc.get_writer('EXAMPLE', 'COMMONCRAWL', warcinfo, warc_version='1.1') as writer: - for obj in cdx.iter(url, limit=10): - url = obj['url'] - status = obj['status'] - timestamp = obj['timestamp'] - - print('considering extracting url', url, 'timestamp', timestamp) - if status != '200': - print(' skipping because status was {}, not 200'.format(status)) - continue - - try: - record = obj.fetch_warc_record() - except RuntimeError: - print(' skipping capture for RuntimeError 404: %s %s', url, timestamp) - continue - writer.write_record(record) - - print(' wrote', url) +result = repackage( + captures, + prefix='EXAMPLE', + subprefix='COMMONCRAWL', + is_part_of='EXAMPLE-COMMONCRAWL', + description='warc extraction', +) +print('wrote', result.written, 'captures to EXAMPLE-COMMONCRAWL-001.warc.gz, ...;', result.failed, 'failed') diff --git a/requirements.txt b/requirements.txt index dba64ea..f40d819 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,10 +3,28 @@ requests>=2.25.1 warcio==1.7.4 +fsspec +aiofiles +url-is-in>=0.1.1 # optional S3 dependencies (install via cdx_toolkit[s3]) fsspec[s3] botocore +aioboto3 + +# optional Athena dependencies (install via cdx_toolkit[athena]) +boto3 + +# optional DuckDB dependencies (install via cdx_toolkit[duckdb]) +duckdb +pytz + +# optional Hugging Face dependencies (install via cdx_toolkit[hf]) +huggingface_hub>=1.7.2 +aiohttp + +# optional Parquet dependencies (install via cdx_toolkit[parquet]) +pyarrow # used by Makefile pytest>=6.2.4 @@ -15,6 +33,7 @@ pytest-sugar>=0.9.4 coveralls>=3.1.0 flake8>=7.3.0 responses==0.25.8 +moto[server]>=5 pre-commit>=4.3.0 # packaging diff --git a/scripts/cdx_iter b/scripts/cdx_iter deleted file mode 100644 index 99445c0..0000000 --- a/scripts/cdx_iter +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python - -import logging -import argparse -import json -import csv -import sys -import os.path - -import cdx_toolkit -import cdx_toolkit.warc - -print('This command is deprecated in favor of cdxt', file=sys.stderr) - -ARGS = argparse.ArgumentParser(description='cdx_toolkit iterator command line tool') -ARGS.add_argument('--cc', action='store_const', const='cc', help='direct the query to the Common Crawl CDX/WARCs') -ARGS.add_argument('--ia', action='store_const', const='ia', help='direct the query to the Internet Archive CDX/wayback') -ARGS.add_argument('--source', action='store', help='direct the query to this CDX server') -ARGS.add_argument('--wb', action='store', help='direct replays for content to this wayback') -ARGS.add_argument('--limit', type=int, action='store') -ARGS.add_argument('--cc-mirror', action='store', help='use this Common Crawl index mirror') -ARGS.add_argument('--cc-sort', action='store', help='default mixed, alternatively: ascending') -ARGS.add_argument('--from', action='store') -ARGS.add_argument('--to', action='store') -ARGS.add_argument('--closest', action='store') # this is a get() thing, not an iter thing -ARGS.add_argument('--filter', action='store', help='see CDX API documentation for usage') -ARGS.add_argument('--all-fields', action='store_true') -ARGS.add_argument('--fields', action='store', default='url,status,timestamp', help='try --all-fields if you need the list') -ARGS.add_argument('--jsonl', action='store_true') -ARGS.add_argument('--csv', action='store_true') -ARGS.add_argument('--warc', action='store_true', help='create a warc of the output contents') -ARGS.add_argument('--warc-prefix', default='TEST', help='prefix for the warc filename') -ARGS.add_argument('--warc-subprefix', type=str, default=None, help='subprefix for the warc filename, default None') -ARGS.add_argument('--warc-size', type=int, default=1000000000, help='target for the warc filesize in bytes') -ARGS.add_argument('--warc-creator', action='store', help='creator of the warc: person, organization, service') -ARGS.add_argument('--warc-operator', action='store', help='a person, if the creator is an organization') -ARGS.add_argument('--warc-url-fgrep', action='store', help='this pattern must be present to warc an url') -ARGS.add_argument('--warc-url-fgrepv', action='store', help='this pattern must not be present to warc an url, e.g. /robots.txt') -ARGS.add_argument('--get', action='store_true', help='use a single get instead of a paged iteration') -ARGS.add_argument('--verbose', '-v', action='count', help='set logging level to INFO (-v) or DEBUG (-vv)') -ARGS.add_argument('url') - -cmdline = ' '.join(sys.argv[1:]) -args = ARGS.parse_args() - -loglevel = os.getenv('LOGLEVEL') or 'WARNING' -if args.verbose: - if args.verbose > 0: - loglevel = 'INFO' - if args.verbose > 1: - loglevel = 'DEBUG' -logging.basicConfig(level=loglevel) -LOGGER = logging.getLogger(__name__) - -if not args.url: - raise ValueError('must specify an url to iterate, example: commoncrawl.org/*') - -kwargs = {} -kwargs['source'] = args.cc or args.ia or args.source or None -if kwargs['source'] is None: - raise ValueError('must specify --cc, --ia, or a --source') -if args.wb: - kwargs['wb'] = args.wb -if args.cc_mirror: - kwargs['cc_mirror'] = args.cc_mirror - -cdx = cdx_toolkit.CDXFetcher(**kwargs) - -fields = set(args.fields.split(',')) - -kwargs = {} -if args.limit: - kwargs['limit'] = args.limit -if 'from' in vars(args) and vars(args)['from']: # python, uh, from is a reserved word - kwargs['from_ts'] = vars(args)['from'] -if args.to: - kwargs['to'] = args.to -if args.closest: - if not args.get: - LOGGER.info('note: --closest works best with --get') - kwargs['closest'] = args.closest -if args.filter: - kwargs['filter'] = args.filter - -if args.csv: - writer = csv.DictWriter(sys.stdout, fieldnames=sorted(list(fields))) - writer.writeheader() - - -def winnow_fields(obj): - if args.all_fields: - printme = obj - else: - printme = dict([(k, obj[k]) for k in fields if k in obj]) - return printme - - -def print_line(printme): - if args.jsonl: - print(json.dumps(printme, sort_keys=True)) - elif args.csv: - writer.writerow(printme) - else: - print(', '.join([' '.join((k, printme[k])) for k in sorted(printme.keys())])) - - -if args.get: - objs = cdx.get(args.url, **kwargs) - for obj in objs: - printme = winnow_fields(obj) - print_line(printme) -elif args.warc: - info = { - 'software': 'pypi_cdx_toolkit/'+cdx_toolkit.__version__, - 'isPartOf': args.warc_prefix, - 'description': 'warc extraction generated with: cdx_toolkit '+cmdline, - 'format': 'WARC file version 1.0', # todo: if we directly read a warc, have this match the warc - } - if args.warc_creator: - info['creator'] = args.warc_creator - if args.warc_operator: - info['operator'] = args.warc_operator - - writer_kwargs = {} - writer_kwargs['size'] = args.warc_size - - writer = cdx_toolkit.warc.get_writer(args.warc_prefix, args.warc_subprefix, info, **writer_kwargs) - - for obj in cdx.iter(args.url, **kwargs): - url = obj['url'] - if args.warc_url_fgrep and args.warc_url_fgrep not in url: - LOGGER.debug('not warcing due to fgrep: %s', url) - continue - if args.warc_url_fgrepv and args.warc_url_fgrepv in url: - LOGGER.debug('not warcing due to fgrepv: %s', url) - continue - timestamp = obj['timestamp'] - try: - record = obj.fetch_warc_record() - except RuntimeError: # pragma: no cover - LOGGER.warning('skipping capture for RuntimeError 404: %s %s', url, timestamp) - continue - if obj.is_revisit(): - LOGGER.warning('revisit record being resolved for url %s %s', url, timestamp) - writer.write_record(record) - - writer.close() -else: - for obj in cdx.iter(args.url, **kwargs): - printme = winnow_fields(obj) - print_line(printme) diff --git a/scripts/cdx_size b/scripts/cdx_size deleted file mode 100644 index 1135f80..0000000 --- a/scripts/cdx_size +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python - -import os -import logging -import argparse -import sys - -import cdx_toolkit - -print('This command is deprecated in favor of cdxt', file=sys.stderr) - -loglevel = os.getenv('LOGLEVEL') or 'WARNING' -logging.basicConfig(level=loglevel) - -ARGS = argparse.ArgumentParser(description='cdx_toolkit size command line tool') -ARGS.add_argument('--cc', action='store_true') -ARGS.add_argument('--ia', action='store_true') -ARGS.add_argument('url') - -args = ARGS.parse_args() - -if not (args.cc or args.ia): - raise ValueError('must specify one of --cc or --ia') - -if args.cc: - cdx = cdx_toolkit.CDXFetcher(source='cc') - print('cc size estimate', cdx.get_size_estimate(args.url)) - -if args.ia: - cdx = cdx_toolkit.CDXFetcher(source='ia') - print('ia size estimate', cdx.get_size_estimate(args.url)) diff --git a/setup.py b/setup.py index b707eb6..9dfec6f 100755 --- a/setup.py +++ b/setup.py @@ -8,10 +8,21 @@ packages = find_packages(include=['cdx_toolkit*']) # remember: keep requires synchronized with requirements.txt -requires = ['requests', 'warcio'] +requires = ['requests', 'warcio', 'fsspec', 'aiofiles', 'url-is-in>=0.1.1'] test_requirements = ['pytest', 'pytest-cov', 'flake8', 'responses'] -optional_s3_requirements = ['fsspec[s3]', 'botocore'] +# S3 mocked with a local server in the S3 tests (not in `test`: it installs boto3 for the minimal tests) +test_s3_requirements = ['moto[server]'] +# S3 reads/writes (index files, WARCs) +optional_s3_requirements = ['fsspec[s3]', 'botocore', 'aioboto3'] +# `cdxt index athena://...` +optional_athena_requirements = ['boto3'] +# `cdxt index cc-url-index-s3` and other parquet indexes +optional_duckdb_requirements = ['duckdb', 'pytz'] # pytz: DuckDB returns timestamps with time zone +# Hugging Face Storage Buckets (hf://): `cdxt index cc-url-index-hf`, `cdxt repackage` reads and writes +optional_hf_requirements = ['huggingface_hub>=1.7.2', 'aiohttp'] +# Parquet output of `cdxt index` (and Parquet input of `cdxt repackage`) +optional_parquet_requirements = ['pyarrow'] package_requirements = ['twine', 'setuptools', 'setuptools-scm'] @@ -19,14 +30,20 @@ extras_require = { 's3': optional_s3_requirements, + 'athena': optional_athena_requirements, + 'duckdb': optional_duckdb_requirements, + 'hf': optional_hf_requirements, + 'parquet': optional_parquet_requirements, 'test': test_requirements, # setup no longer tests, so make them an extra 'package': package_requirements, 'dev': package_requirements, - 'all': test_requirements + package_requirements + dev_requirements + optional_s3_requirements, + 'all': ( + test_requirements + test_s3_requirements + package_requirements + dev_requirements + + optional_s3_requirements + optional_athena_requirements + + optional_duckdb_requirements + optional_hf_requirements + optional_parquet_requirements + ), } -scripts = ['scripts/cdx_size', 'scripts/cdx_iter'] - this_directory = path.abspath(path.dirname(__file__)) with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: description = f.read() @@ -39,7 +56,7 @@ long_description_content_type='text/markdown', author='Greg Lindahl and others', author_email='lindahl@pbm.com', - url='https://github.com/cocrawler/cdx_toolkit', + url='https://github.com/commoncrawl/cdx_toolkit', packages=packages, python_requires=">=3.9", extras_require=extras_require, @@ -48,9 +65,7 @@ entry_points=''' [console_scripts] cdxt = cdx_toolkit.cli:main - #ccathena = cdx_toolkit.cli:main_athena ''', - scripts=scripts, license='Apache 2.0', classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/conftest.py b/tests/conftest.py index 9f54f82..b124016 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,37 @@ +"""Shared test fixtures. + +By default, the tests make no remote requests: +- the CDX API and HTTP requests (data.commoncrawl.org) are mocked with `responses` and recorded data + (see mock_response_from_jsonl) +- S3 is a local moto server (see the s3_env fixture and requires_aws_s3) +- Hugging Face reads are faked in the tests that need them + +CDXT_LIVE_TESTS is a comma-separated list of the services to test live instead (see LIVE_SERVICES), or `all`: +- http: the CDX API and HTTP requests go to the real servers +- s3: the S3 tests run against AWS, and the tests of DuckDB over s3://commoncrawl run (requires_live_s3) +- athena: the Athena tests run (requires_aws_athena) +- hf: the tests of the Hugging Face Storage Bucket hf://buckets/commoncrawl/commoncrawl run (requires_live_hf) + +The tests that can run live are marked `live` and `live_` (see live_marks), e.g. +`CDXT_LIVE_TESTS=all pytest -m live` runs only the live tests, `CDXT_LIVE_TESTS=s3 pytest -m live_s3` only those of S3. +Without CDXT_LIVE_TESTS, the marked tests are skipped (s3, athena, hf) or run with mocked responses (http, s3). +""" +import gzip import json import os +import re +import uuid +import urllib.request from pathlib import Path import pytest -from cdx_toolkit.settings import CACHE_DIR + +try: + import boto3 + _HAS_BOTO3 = True +except ImportError: # pragma: no cover - exercised in minimal installs + boto3 = None + _HAS_BOTO3 = False try: import botocore.session @@ -22,30 +50,99 @@ except ImportError: # pragma: no cover - exercised in minimal installs _HAS_FSSPEC = False +try: + import duckdb + _HAS_DUCKDB = True +except ImportError: # pragma: no cover - exercised in minimal installs + _HAS_DUCKDB = False + +try: + import huggingface_hub # noqa: F401 + _HAS_HF = True +except ImportError: # pragma: no cover - exercised in minimal installs + _HAS_HF = False + +try: + import aioboto3 # noqa: F401 + _HAS_AIOBOTO3 = True +except ImportError: # pragma: no cover - exercised in minimal installs + _HAS_AIOBOTO3 = False + +try: + import moto.server # noqa: F401 + import s3fs as _s3fs # noqa: F401 + _HAS_MOTO = _HAS_AIOBOTO3 +except ImportError: # pragma: no cover - exercised in minimal installs + _HAS_MOTO = False + import functools from typing import Dict, Optional import requests import responses import base64 -import shutil from unittest.mock import patch - +TEST_DATA_PATH = Path(__file__).parent / 'data' TEST_DATA_BASE_PATH = Path(__file__).parent / 'data' TEST_S3_BUCKET = os.environ.get('CDXT_TEST_S3_BUCKET', 'commoncrawl-ci-temp') -DISABLE_S3_TESTS = bool(os.environ.get('CDXT_DISABLE_S3_TESTS', False)) +TEST_ATHENA_S3_LOCATION = 's3://commoncrawl-ci-temp/athena-results/' +TEST_ATHENA_DATABASE = 'ccindex' +# a writable Hugging Face Storage Bucket (namespace/name) for the hf:// output tests (with CDXT_LIVE_TESTS=hf) +TEST_HF_BUCKET = os.environ.get('CDXT_TEST_HF_BUCKET', 'commoncrawl/commoncrawl-ci-temp') + +LIVE_SERVICES = ('http', 's3', 'athena', 'hf') -# Cache for AWS access check to avoid repeated network calls + +def _live_services(value: str) -> frozenset: + """The services of CDXT_LIVE_TESTS (comma-separated, or `all`).""" + names = {name.strip() for name in value.split(',') if name.strip()} + if 'all' in names: + return frozenset(LIVE_SERVICES) + unknown = names - set(LIVE_SERVICES) + if unknown: + raise ValueError(f'CDXT_LIVE_TESTS: unknown service(s) {", ".join(sorted(unknown))}; ' + f'use a comma-separated list of {", ".join(LIVE_SERVICES)}, or all') + return frozenset(names) + + +LIVE_TESTS = _live_services(os.environ.get('CDXT_LIVE_TESTS', '')) + + +def pytest_configure(config): + config.addinivalue_line('markers', 'live: a test that runs live with CDXT_LIVE_TESTS (any service)') + for service in LIVE_SERVICES: + config.addinivalue_line('markers', f'live_{service}: a test that runs live with CDXT_LIVE_TESTS={service}') + + +def live_marks(func, service): + """Marks a test that runs live with CDXT_LIVE_TESTS=: `live` and `live_`.""" + return pytest.mark.live(getattr(pytest.mark, f'live_{service}')(func)) + + +def pytest_collection_modifyitems(config, items): + """The tests with the mocked data.commoncrawl.org make HTTP requests with CDXT_LIVE_TESTS=http.""" + for item in items: + if 'mock_data_commoncrawl' in item.fixturenames and not item.get_closest_marker('live_http'): + item.add_marker(pytest.mark.live) + item.add_marker(pytest.mark.live_http) + + +# Cache for AWS S3/Athena access check to avoid repeated network calls _aws_s3_access_cache = None +_aws_athena_access_cache = None @pytest.fixture(scope='session', autouse=True) -def cleanup_cache(): - """Delete cache directory before each test to ensure clean state""" - cache_dir = os.path.expanduser(CACHE_DIR) - if os.path.exists(cache_dir): - shutil.rmtree(cache_dir) +def cache_dir(tmp_path_factory): + """An empty cache directory for the tests: the user's cache (CDXT_CACHE_DIR) keeps its list of crawls, and gets + none of the recorded ones. Also set in the environment, for worker processes.""" + cache = str(tmp_path_factory.mktemp('cdxt-cache')) + '/' + with pytest.MonkeyPatch.context() as mp: + mp.setattr('cdx_toolkit.settings.CACHE_DIR', cache) + mp.setattr('cdx_toolkit.commoncrawl.CACHE_DIR', cache) + mp.setenv('CDXT_CACHE_DIR', cache) + yield cache @pytest.fixture(scope='session', autouse=True) @@ -80,18 +177,319 @@ def check_aws_s3_access(): return _aws_s3_access_cache -def requires_aws_s3(func): - """Pytest decorator that skips test if AWS S3 access is not available or disabled.""" +def requires_live_s3(func): + """Pytest decorator for tests that need AWS S3 itself (not mockable, e.g. DuckDB over s3://commoncrawl): + they only run with CDXT_LIVE_TESTS=s3 and AWS S3 access.""" + func = live_marks(func, 's3') + if 's3' not in LIVE_TESTS: + return pytest.mark.skip(reason='needs AWS S3; set CDXT_LIVE_TESTS=s3 to run it')(func) if not _HAS_BOTOCORE or not _HAS_FSSPEC: + return pytest.mark.skip(reason='S3 dependencies are not installed; install cdx_toolkit[s3]')(func) + return pytest.mark.usefixtures('live_aws_s3_access')(func) + + +@pytest.fixture +def live_aws_s3_access(): + """Skip the test without AWS S3 access; checked when a test needs it (not when the tests are collected).""" + if not check_aws_s3_access(): + pytest.skip('AWS S3 access not available (no credentials or permissions)') + + +def requires_aws_s3(func): + """Pytest decorator for S3 tests: they run against a local moto server (the s3_env fixture), or against + AWS with CDXT_LIVE_TESTS=s3.""" + func = live_marks(func, 's3') + if 's3' in LIVE_TESTS: + return pytest.mark.usefixtures('s3_env')(requires_live_s3(func)) + if not _HAS_MOTO: + return pytest.mark.skip(reason='moto, s3fs and aioboto3 are needed to mock S3; install cdx_toolkit[all]')(func) + return pytest.mark.usefixtures('s3_env')(func) + + +def check_aws_athena_access(): + """Check if AWS Athena access is available.""" + global _aws_athena_access_cache + + if not _HAS_BOTO3: + return False + + if _aws_athena_access_cache is not None: + return _aws_athena_access_cache + + try: + boto3.client('athena').list_databases(CatalogName='AwsDataCatalog') + _aws_athena_access_cache = True + except (NoCredentialsError, ClientError): + _aws_athena_access_cache = False + + return _aws_athena_access_cache + + +def requires_aws_athena(func): + """Pytest decorator that skips test if AWS Athena access is not available (needs CDXT_LIVE_TESTS=athena).""" + func = live_marks(func, 'athena') + if 'athena' not in LIVE_TESTS: + return pytest.mark.skip(reason='needs AWS Athena; set CDXT_LIVE_TESTS=athena to run it')(func) + if not _HAS_BOTO3: return pytest.mark.skipif( - True, reason='S3 dependencies are not installed; install warcio[s3] to enable S3 tests.' + True, reason='S3 dependencies are not installed; install cdx_toolkit[s3] to enable Athena tests.' )(func) - return pytest.mark.skipif(DISABLE_S3_TESTS, reason='AWS S3 access is disabled via environment variable.')( - pytest.mark.skipif( - not check_aws_s3_access(), reason='AWS S3 access not available (no credentials or permissions)' - )(func) - ) + return pytest.mark.usefixtures('live_aws_athena_access')(func) + + +@pytest.fixture +def live_aws_athena_access(): + """Skip the test without AWS Athena access; checked when a test needs it (not when the tests are collected).""" + if not check_aws_athena_access(): + pytest.skip('AWS Athena access not available (no credentials or permissions)') + + +def requires_live_hf(func): + """Pytest decorator for tests of the Hugging Face Storage Bucket hf://buckets/commoncrawl/commoncrawl: they only + run with CDXT_LIVE_TESTS=hf (public reads need no token, but have lower rate limits without one).""" + func = live_marks(func, 'hf') + if 'hf' not in LIVE_TESTS: + return pytest.mark.skip(reason='needs Hugging Face; set CDXT_LIVE_TESTS=hf to run it')(func) + if not _HAS_HF: + return pytest.mark.skip(reason='huggingface_hub is not installed; install cdx_toolkit[hf]')(func) + return func + + +def requires_duckdb(func): + """Pytest decorator that skips a test if the optional duckdb dependency is missing.""" + return pytest.mark.skipif( + not _HAS_DUCKDB, reason='duckdb is not installed; install cdx_toolkit[duckdb] to enable DuckDB tests.' + )(func) + + +def requires_aioboto3(func): + """Pytest decorator that skips a test if the optional aioboto3 dependency is missing (no AWS access needed).""" + return pytest.mark.skipif( + not _HAS_AIOBOTO3, reason='aioboto3 is not installed; install cdx_toolkit[s3] to enable these tests.' + )(func) + + +# The WARC records of the first 10 captures of tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.gz, +# recorded from data.commoncrawl.org: {(filename, offset, length): bytes} +WARC_RECORDS = {} +with open(TEST_DATA_PATH / 'repackage' / 'warc_records.jsonl') as _f: + for _line in _f: + _r = json.loads(_line) + WARC_RECORDS[(_r['filename'], _r['offset'], _r['length'])] = base64.b64decode(_r['data']) + + +def _data_commoncrawl_range(request): + """responses callback: a range request to data.commoncrawl.org, served from WARC_RECORDS.""" + filename = request.url.split('://', 1)[1].split('/', 1)[1] + start, end = (int(n) for n in request.headers['Range'][len('bytes='):].split('-')) + data = WARC_RECORDS.get((filename, start, end - start + 1)) + if data is None: + return 404, {}, f'no recorded WARC record for {filename} bytes {start}-{end} (see tests/conftest.py)' + return 206, {'Content-Range': f'bytes {start}-{end}/*'}, data + + +@pytest.fixture +def mock_data_commoncrawl(): + """WARC record downloads from data.commoncrawl.org are served from recorded data (unless + CDXT_LIVE_TESTS=http); other requests made with `requests` fail, unless the test adds mock data (e.g. + with mock_response_from_jsonl).""" + if 'http' in LIVE_TESTS: + yield None + return + responses.start() + try: + responses.add_callback(responses.GET, re.compile(r'https://data\.commoncrawl\.org/.*'), + callback=_data_commoncrawl_range) + yield responses.mock + finally: + responses.stop() + responses.reset() + + +# crawl and host of the captures of WARC_RECORDS +LOCAL_INDEX_CRAWL = 'CC-MAIN-2024-30' +LOCAL_INDEX_HOST = 'bibliotheque.missiondefrance.fr' + + +@pytest.fixture +def local_url_index(tmp_path): + """A columnar URL index like Common Crawl's (/crawl=.../subset=warc/*.parquet) with the captures of + WARC_RECORDS, and captures that are not selected by the tests: of another host, and of another crawl.""" + from cdx_toolkit.record_io import parse_cdxj_line + + with gzip.open(TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz', 'rt') as f: + captures = [parse_cdxj_line(f.readline()) for _ in range(len(WARC_RECORDS))] + rows = {LOCAL_INDEX_CRAWL: captures + [dict(captures[0], url='https://example.com/', urlkey='com,example)/')], + 'CC-MAIN-2024-26': [captures[0]]} + + base = tmp_path / 'url-index' + con = duckdb.connect() + for crawl, crawl_rows in rows.items(): + path = base / f'crawl={crawl}' / 'subset=warc' + path.mkdir(parents=True) + con.execute('''CREATE OR REPLACE TABLE idx ( + url_surtkey VARCHAR, url VARCHAR, url_host_name VARCHAR, url_host_name_reversed VARCHAR, + url_host_registered_domain VARCHAR, url_host_tld VARCHAR, fetch_time TIMESTAMP, fetch_status SMALLINT, + content_mime_type VARCHAR, content_mime_detected VARCHAR, content_digest VARCHAR, content_languages VARCHAR, + content_charset VARCHAR, fetch_redirect VARCHAR, warc_filename VARCHAR, warc_record_offset INTEGER, + warc_record_length INTEGER)''') + for c in crawl_rows: + host = c['url'].split('/')[2] + values = "?, ?, ?, ?, ?, ?, strptime(?, '%Y%m%d%H%M%S'), ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?" + con.execute(f'INSERT INTO idx VALUES ({values})', [ + c['urlkey'], c['url'], host, '.'.join(reversed(host.split('.'))), '.'.join(host.split('.')[-2:]), + host.split('.')[-1], c['timestamp'], + int(c['status']), c['mime'], c['mime-detected'], c['digest'], c['languages'], c.get('charset'), + c['filename'], int(c['offset']), int(c['length']), + ]) + con.execute(f"COPY idx TO '{path}/part-00000.parquet' (FORMAT parquet)") + return base + + +class InlineExecutor: + """ProcessPoolExecutor stand-in that runs the workers in this process (mocks apply, coverage is measured).""" + + def __init__(self, max_workers=None, mp_context=None, initializer=None, initargs=()): + if initializer: + initializer(*initargs) + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def map(self, fn, items): + return [fn(item) for item in items] + + def submit(self, fn, *args): + import concurrent.futures + + future = concurrent.futures.Future() + try: + future.set_result(fn(*args)) + except Exception as e: + future.set_exception(e) + return future + + def shutdown(self, wait=True, cancel_futures=False): + pass + + +class S3Env: + """S3 for a test: a local moto server with the buckets commoncrawl (the WARC files of WARC_RECORDS) and + TEST_S3_BUCKET, or AWS itself (CDXT_LIVE_TESTS=s3).""" + + def __init__(self, offsets=None): + # (filename, offset) -> offset in the mocked WARC file, which holds only the recorded records + self.offsets = offsets + + def capture(self, capture): + """The capture with the offset of its record in s3://commoncrawl (the mocked files are short).""" + if self.offsets is None: + return capture + offset = self.offsets[(capture['filename'], int(capture['offset']))] + return dict(capture, offset=type(capture['offset'])(offset)) + + +@pytest.fixture(scope='session') +def moto_server(): + import logging + from moto.server import ThreadedMotoServer + + logging.getLogger('werkzeug').setLevel(logging.WARNING) + server = ThreadedMotoServer(ip_address='127.0.0.1', port=0, verbose=False) + server.start() + host, port = server.get_host_and_port() + yield f'http://{host}:{port}' + server.stop() + + +@pytest.fixture +def s3_env(request, monkeypatch, tmp_path): + """S3 for a test, see S3Env. The moto server is configured with environment variables only, so that also + worker processes (cdxt repackage --processes) use it.""" + if 's3' in LIVE_TESTS: + yield S3Env() + return + + import fsspec.config + import s3fs + from cdx_toolkit import aws + + endpoint = request.getfixturevalue('moto_server') + # path-style addressing: the endpoint is an IP address (an explicit S3 setting of the AWS config file wins + # over cdxt's defaults) + config_file = tmp_path / 'aws-config' + config_file.write_text('[default]\nregion = us-east-1\ns3 =\n addressing_style = path\n') + for name in ('AWS_PROFILE', 'AWS_SESSION_TOKEN', 'AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_MAX_ATTEMPTS', + 'AWS_RETRY_MODE', 'CDXT_AWS_DEFAULTS'): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv('AWS_ENDPOINT_URL_S3', endpoint) + monkeypatch.setenv('AWS_CONFIG_FILE', str(config_file)) + monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', str(tmp_path / 'aws-credentials')) + monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'testing') + monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'testing') + monkeypatch.setitem(fsspec.config.conf, 's3', {}) + aws.user_aws_config.cache_clear() + s3fs.S3FileSystem.clear_instance_cache() + + urllib.request.urlopen(urllib.request.Request(f'{endpoint}/moto-api/reset', method='POST')) + client = boto3.client('s3', endpoint_url=endpoint, region_name='us-east-1') + client.create_bucket(Bucket='commoncrawl') + client.create_bucket(Bucket=TEST_S3_BUCKET) + offsets, files = {}, {} + for (filename, offset, _), data in sorted(WARC_RECORDS.items()): + offsets[(filename, offset)] = len(files.get(filename, b'')) + files[filename] = files.get(filename, b'') + data + for filename, data in files.items(): + client.put_object(Bucket='commoncrawl', Key=filename, Body=data) + + yield S3Env(offsets) + + aws.user_aws_config.cache_clear() + s3fs.S3FileSystem.clear_instance_cache() + + +@pytest.fixture +def s3_tmpdir(s3_env): + """S3 equivalent of tmpdir - provides a temporary S3 path and handles cleanup.""" + bucket_name = TEST_S3_BUCKET + + # Generate unique prefix using UUID to avoid collisions + temp_prefix = f'cdx_toolkit/ci/tmpdirs/{uuid.uuid4().hex}' + + # Yield the S3 path + yield f's3://{bucket_name}/{temp_prefix}' + + try: + # Cleanup: delete all objects with this prefix + s3_client = boto3.client('s3') + + # List all objects with the temp prefix + response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=temp_prefix) + + if 'Contents' in response: + # Delete all objects + objects_to_delete = [{'Key': obj['Key']} for obj in response['Contents']] + s3_client.delete_objects(Bucket=bucket_name, Delete={'Objects': objects_to_delete}) + except (NoCredentialsError, ClientError, ConnectionError, EndpointConnectionError): + # Ignore cleanup errors - test objects will eventually expire + pass + + +@pytest.fixture +def hf_tmpdir(): + """A temporary path in the Hugging Face Storage Bucket TEST_HF_BUCKET (CDXT_LIVE_TESTS=hf), removed afterwards.""" + from huggingface_hub import HfFileSystem + + path = f'buckets/{TEST_HF_BUCKET}/cdx_toolkit/ci/tmpdirs/{uuid.uuid4().hex}' + yield f'hf://{path}' + try: + HfFileSystem().rm(path, recursive=True) + except Exception: # noqa: BLE001 - cleanup only; nothing written, or no permission + pass def flexible_param_matcher(expected_params): @@ -141,7 +539,7 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None (if you run this from a whitelisted IP address), the following environment variable can be set: ```bash - export DISABLE_MOCK_RESPONSES=1 + export CDXT_LIVE_TESTS=http ``` If the remote APIs change, new mock data can be semi-automatically collected @@ -149,10 +547,10 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None and overwriting existing mock data in `tests/data/mock_responses`: ```bash - # set environment variable (DISABLE_MOCK_RESPONSES should not be set) - export SAVE_MOCK_RESPONSES=./tmp/mock_responses + # set environment variable (CDXT_LIVE_TESTS should not have http) + export CDXT_SAVE_MOCK_RESPONSES=./tmp/mock_responses - # run the test for what mock data should be saved to $SAVE_MOCK_RESPONSES//.jsonl + # run the test for what mock data should be saved to $CDXT_SAVE_MOCK_RESPONSES//.jsonl pytest tests/test_cli.py::test_basics ``` @@ -195,28 +593,39 @@ def mock_response_from_jsonl(mock_data_name, mock_data_dir: Optional[str] = None ) -def conditional_mock_responses(func): - """Conditionally applies @responses.activate and auto-loads mock data based on DISABLE_MOCK_RESPONSES env var. +def conditional_mock_responses(func=None, *, auto_mock_data: bool = True): + """Applies @responses.activate and auto-loads mock data, unless CDXT_LIVE_TESTS has http. The mock data is automatically loaded from JSONL file from the tests/data directory and dependinng on the test module and test function. + + Args: + auto_mock_data: If True, auto-loads test-specific mock data. If False, only loads CC endpoints. """ - # If the flag DISABLE_MOCK_RESPONSES is not detected, response mocking remains enabled - if not os.environ.get('DISABLE_MOCK_RESPONSES'): - # Add responses.activate - func = add_mock_responses(func) + def decorator(f): + # without CDXT_LIVE_TESTS=http, the responses are mocked + if 'http' not in LIVE_TESTS: + # Add responses.activate + f = add_mock_responses(f, auto_mock_data=auto_mock_data) - if os.environ.get('SAVE_MOCK_RESPONSES'): - # Mock data is saved by capturing output from requests.get - @functools.wraps(func) - def wrapper(*args, **kwargs): - with patch('requests.get', side_effect=_custom_behavior_with_original(requests.get)): - return func(*args, **kwargs) + if os.environ.get('CDXT_SAVE_MOCK_RESPONSES'): + # Mock data is saved by capturing output from requests.get + @functools.wraps(f) + def wrapper(*args, **kwargs): + with patch('requests.get', side_effect=_custom_behavior_with_original(requests.get)): + return f(*args, **kwargs) - return wrapper + return live_marks(wrapper, 'http') - return func + return live_marks(f, 'http') + + if func is None: + # Called with arguments: @conditional_mock_responses(auto_mock_data=False) + return decorator + else: + # Called without arguments: @conditional_mock_responses + return decorator(func) def save_response_as_mock_data(test_info: str, request_url: str, request_params: Dict, resp, output_base_dir: str): @@ -275,7 +684,7 @@ def custom_behavior(*args, **kwargs): request_url=request_url, request_params=request_params, resp=response, - output_base_dir=os.environ.get('SAVE_MOCK_RESPONSES'), + output_base_dir=os.environ.get('CDXT_SAVE_MOCK_RESPONSES'), ) return response @@ -283,14 +692,15 @@ def custom_behavior(*args, **kwargs): return custom_behavior -def add_mock_responses(func): +def add_mock_responses(func, auto_mock_data: bool = True): @functools.wraps(func) def wrapper(*args, **kwargs): # Load mock data for index calls (same for many test functions) mock_response_from_jsonl('test_get_cc_endpoints', 'test_cc') # Auto-load mock data based on function name - mock_response_from_jsonl(func.__name__, func.__module__.split('.')[-1]) + if auto_mock_data: + mock_response_from_jsonl(func.__name__, func.__module__.split('.')[-1]) return func(*args, **kwargs) return responses.activate(wrapper) diff --git a/tests/data/mock_responses/test_cli/test_multi_misc_not_ia_slow_5.jsonl b/tests/data/mock_responses/test_cli/test_multi_misc_not_ia_slow_5.jsonl deleted file mode 100644 index 7bd7eab..0000000 --- a/tests/data/mock_responses/test_cli/test_multi_misc_not_ia_slow_5.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"method": "GET", "url": "https://index.commoncrawl.org/CC-MAIN-2025-33-index", "request_params": {"url": "commoncrawl.org/*", "showNumPages": "true", "limit": 10, "from": "20240915203928"}, "response_status_code": 200, "response_text": "{\"pages\": 1, \"pageSize\": 5, \"blocks\": 1}", "response_headers": {"Server": "nginx/1.29.1", "Date": "Mon, 15 Sep 2025 20:39:30 GMT", "Content-Type": "text/x-cdxj", "Transfer-Encoding": "chunked", "Connection": "keep-alive"}} diff --git a/tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.gz b/tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.gz new file mode 100644 index 0000000..47941ed Binary files /dev/null and b/tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.gz differ diff --git a/tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.metadata.json b/tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.metadata.json new file mode 100644 index 0000000..6a4103f --- /dev/null +++ b/tests/data/repackage/filtered_CC-MAIN-2024-30_cdx-00187.metadata.json @@ -0,0 +1,4 @@ +{ + "uri": "filter_cdx.cdx.gz", + "warc_content_type": "application/cdx" +} \ No newline at end of file diff --git a/tests/data/repackage/url_list.txt b/tests/data/repackage/url_list.txt new file mode 100644 index 0000000..475f73d --- /dev/null +++ b/tests/data/repackage/url_list.txt @@ -0,0 +1,10 @@ +example.com +si.edu +youtube.com +archive.gov +census.gov +onlinedegrees.741.com/online_university_degree_program.html +star.72pines.com/2007/06/25/%e6%8f%90%e5%8f%96%e5%85%ac%e7%a7%af%e9%87%91/trackback +bibliotheque.missiondefrance.fr +biodiv.mnhn.fr/fr/taxonomy +wip.mobilierpourchr.fr/produit/t-837 diff --git a/tests/data/repackage/warc_records.jsonl b/tests/data/repackage/warc_records.jsonl new file mode 100644 index 0000000..ef6822e --- /dev/null +++ b/tests/data/repackage/warc_records.jsonl @@ -0,0 +1,10 @@ +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514759.37/warc/CC-MAIN-20240716142214-20240716172214-00337.warc.gz", "offset": 111440525, "length": 9754, "data": "H4sIAAAAAAAAAO2d23LbSNKgr62IfodqTtiQt80DDiQAydQELcndcus0ltzdMx0djAJQpGCDABsHye6JjtgH2Yu92r8d/xNs7KXeZO/2LTazCgBJERRJkZT1W+wZSxRQh6ysrMz8qkDy59bb3apcqX2z8TO8Kp9/6rMtErKoH/gRSy/u0RguKjVFK9f0stw4l+tbqrxVr/8rLfCW2UHolA/2tsjLJPS3ksR1tmSVqpbhyGXdZEpZ6zBWNjSblRVN1nTasSxVsXe+2dgN/Jj5cfmQ+d34YovodUOpDy4LgWi/77k2jd3Ar17EcX+b9KJuDLeaN0T9mYa263eCG7LUDaZYms7KOlOdsqbbjbKhm1qZmppZa9TqjqxZO2kT0LOdhCHvPBgZkEZrHWaYoATTKmsNyyhbtYZatmt1jVHbUW3DzBo5OC23HAeEi7aIXDMriqpV5EajouSKpmGXxeV3bw+2CI4o2qpWLdfy3CC+YL8nrNJzowiG67BOSH2bVTph1YW/Plb6F/2/e5de00o8j8Wu33bcqO/RT89cp6nKZtr+aRjEgR14onWYYXnsRuxFcF3Nhu32L1hYPktcnOzzw7P27g8t+L9Sa5+eHP5TVmv19hn8WW9kDdFPXkCd8p7bZVG8RaILKm/t1X9UX7/7ee9t6+To6PzNrrJ7/vNP+qHSevX64E391Wla95UX2B9Ga/7w+vxVXd87OfjX93unZ292f9SP/9E61xqnx/Wjkx/e7mWKdWBi3I7LnFyAcRP5eBH3vO8+9rxvNr7Z+OH8/BTHDwZcIyc/frMhzPk8YS+I3CBvEo+bNkmNWiPfH53ftL+YfYyr2Og2sS9oGLG4mcSdsvHNxi/l3ZBeeaC5c5inqAMv9n07cFy/uwVlE/8Dc3hrPrNRti3ygbF+mXruJdjsTzT8tEVats36cV4PilP7gqEhxiFOVD+I4jJcsT80ay9IP2RDf0y69c3G/se+Cwa4Rc5o/ILUZPKG+qiDGqnVtvj/xUjPwAx3g+CDC+M8vegffXqFZlg+OW3t7r1q2tRmDZz7YePcJkw03uRtK0b5TeKXQYt16GCrnmlxmxzRj+VWlzXVmvjvRm8/nJ7tn50d7DVlymRTUy1mylbDgu5qTLYUxbQMajc6HWub9Gl80azeqN+zTvrULh+efH9w3HQYLAfmDGQ7v0hw3GUYd1k29Xzc8qhsN4VKG0XJjltH+830wuRSIL9hwMprNEyzXlxsr3XeetU6219EyEMKU30EFoLGP9V+z1h4ycItEijRlRvbF+XT4IqFrwK0l0t12G4zSx+YrRUWuGXN1A1cThsvv9072T3/5+k+wQVBTt+9OjzYJaVytfqzulut7p3vkV9+OD86JLDmqtX949IGgf9K6IbAx11dXVWu1EoQdqvnb8VClWWslL2uOLFT2tl4yRuHJexHzYK6smmaokYJC2151O82S52wlK9PiS9QCVti1NnZePKyx2LKvWEZbNi9bJbsdIwYSKCi+KtZmrTYS6SaN+PTHmuWaBJfBOFQ1dOjV+T7MEj6vOxo4Q/s0xVEyWioeOrjicOI8PIvyPAyG+/QYZEdun10JEPNvErrPGPdkF6ybahKjgZNv+ZNF0gUBlYQD8tDPS/t89ty+WCfsJhE0Abzwa1hS3bQg3XoWq7nxv+vXC5SqvRL+V0LbEoU9JiUty4d7Df3nS6Tbozp0mVX/SCMh+S4ch1Y7Q67dCFf4H+8IK7vxi71ypFNPdaUX5Ae/ej2kl52AWzmyZMn0HTsxh7b2aUx9YIuaIL5xHO7foFKXlZF2Y0n+L+Xnut/gNTHa0pR/Mlj0QVjsUR4liFxm7CjSCIXIes0pUpVlKmCSnoQccSvCpT4u6wrtYapaI2aJHS5SLsdUEmZXrEo6LHltx70mV+OIGwtv+mQp4RZbrLE9vs9S2v3nA505GFUHZVdN5bSdhdWAgtd1u5fQLa0ki6yFytpO3T9eBUNx7S7EmVcurgyfZZErLh58EcEbrTR42ICKBI9Aj5ouOuzH07enu++OycHuyfHWW9uj0KaWY0gr612KLgUWKfwg7sh4U2FmMLvv6eXVFwtkSi0myXXt70E/O7QnaoT0m4bEu8w6FfeR6Wdl1VxY4EmL6jveGypTfaDfnJ7Y9KNxlAnhDxxO2TzWyewkx6ougKYsu8xfBm9+rTr0Sg6Br/9/N+kWoX0MwmJy9DzTilPmgRLdRKfp8GbftBr23jz+b/x+pNLCg3FrBc1fXZFWmFIP20+385v2UECgaEmLnSCkGzyCnCFuC+Luz6nXex4U/pv0vOKx9MYKPzdd8/JvwmX5Qkf6AyVUa5N93nFHgymSQYDIGIE0BzK/ysX9bvvfoMRz972tpAIM6U/xSvxK2Qx8KfQDB88v47/QPuvA6FMQkkSQ0yOXAjTfEo8iWE2B4YP0S4i794eRhgHBzO9MZgKwouyNhTadCBqiglB9YL4u5AAQUI7PBbMiagdM3797xOub41ehxyKS+92NocbbZLSu/PXkFmlRpCOVggEaIxpROBDI0Iw3sSfzIvYaHHIAPpsuIhQ0cDqN+Z0T4VrPngfVB33vRtXwQX14Gac+OAI0l8V3gD4qqkLTGj2fQAZdsftgmZxMH3U0Yl/CDwLtBlCbg8XAZIhtdkiUicsd0IJL7nRHrMSyNE7FNSAV8BlnnrU9d9ECJdZVYCOD+j2tsivXFeYakFD4HBf0YjxpnjzHEmlSqXK/y+h4n7Dmw7rY10JqJq76eoRiyJs8CwOQibxMiCcZ0E/W/miHikk5vTK9Z3gqgJN7MGYcbR/8jlKL1R6aRW4gwt/pIXNfyehB9JV6Xsq9j16gZNAsod/PwPhWbeZ1ZdeEAeAyo4PIE/c4tr5k1sDDgmkhV4H9nD7HHGPeqsVZD/AuSJgZO0uo1nQS97yrALvoEkFMOXt6CK4akeBDYly22cx8MYHAj5yZv8vZCySMHJ74GHaEaOhfXFDupkaLR44kkM7RNiJ4putgmIFum28tALnEwl83O5BOuAm5bAOTbz4LKZxEjUlRC++lXL9V0T6CRCRLW2XCHGhBigV1VOaYXXmXrEDvRwHkGmw1HNvug43aVR24jX9xPO2uR+2acS9rM9LR4RHBReHtJH50Z5oqTgkvPp04GxKzJO+c53vpN0L13Ok56Jp0WSWRW9wB/pt3tzz2VoWjYgmuQPOqueOPvEi3kB6Y0K4SjzRxkj8JRiAoX4aYYkIscI/Q2dw51f3N2ywFcehayUxNOTC+ND5S4Hf9Vgctd3IctrpnH2XqjkXDSQTrYjg/8QKGf0gXv+Ze3uSRwWs5Lm36UP0yjvlvWWR6VvPfU5mrj4kbtZAVp/L7LkVcOpQ6xhi2XYatNOIlaDl/DlkbHzdXl3QePP3JHABdh2RVYi9G35RKAVsjRFp/5fTw/bhya5EtrgeJsrquJdt9rHvtcHXS+Q7bLfCA2AlNSkQVLJwM1banq2lia34EKilbe5vb2kmVd5wS0NZFZGGLKHt+pBOuJfztTo80oktZ+1ujNrTQLnLUaxQyYJ6nWl25lHryOCXptLBZN1QauqmU4WiJTcJ/qzEwSFuSe6C0oczfb54Cp3v9kaWsydennTn3EC4n5jgujw3dV0FzovXLnJffE2LnH6aC2tL3+GYhnxXRgNQc7aZmGYkgx5ut5GRDPnGGAaSfDsQHZleIs+ekduLATnPUixKut0Zilk0+gAZ+EBbM6hryB/ktSap7JZpq4QM9GazzMalF6hf6fntCziLNoPffz5J3dKfoxb/J7r2pSRHMYVxR6B6TH/nSQhnaHRp7fXcyB7LB0GhmHhJNMbKqNam9Ef5AM8vt+RarYalocxQUb4ZDxjDwtJOfq0HFwZ/SvhnG1NCFkIDF+pOtvF9/RdueTuMeLRolxdKiu5GGm5fBGHU7gRBjH2mc7uAInBrqp3tuQypA9rMunXxQK+dIks6MG7ukKIqpUzIEVElHxZelwIOIWlEUN/v7rQIpBaxK/bkWRwz+BtQ/zJIIuT/S/YH2YL+ReGNlzjljIhNdQnU/xSki0P456S9i064Pwp80IkbgtWB+Iz5Etl5SQUYlyo3TqD5X6XRNuB1nCbC0EvUp/7OWxYHkPWwkNAEjyh7uE1x/dmG2SH0ktkEeBxGEvru9V/hII+uVCowBmzgZZXCv9hB1EK5uQ2zTFG4NdhhYQ9TK24Ef8tVhvuDufYlaBt8RRtaBAGkTGzW68efpGGtpzPCbeQqBALmNjlZidx20vtpozYNQ5iMiHnt7Jg+XweSsLVsBtKrvOEt7em2lOtbmn7eL9fkOlRwe11C8ZAD1ojrOHjqkhmq2PwErV+6IJBc6ftdqTrQ6JANpM51VCBZrciaYtR18XN+8VRpR2k0SJn85F7C9NgSgB6BNCt0I1izYAf/k9jh9WcWvrRCUt2JWB88N/jScmDHAfwmSk1W7k9cDcXVQdw96kfcMCFS2UmaURAnIT18JuMFB74IryLxOdL+JfW7rgc3WjDrPcuDIZHvrz/7159D6qWj4CP0g0sxQAeWAH91z0Os4xANGOLJ90dAzCAcTA31Y9wg5BwOPkCI+p76l7BUyp3rz5chLloQVC0SNH1ehtt6e2D7S5VclU0U3ATBDynpICV5YDzB9f8m4O198P0YbZjvgC+MhPwUpPbKsI4nCr4aDTdAUL0Ggr52ffRsoGCWajRx/TL8wGplGlz/n/ieRdNRNBlE270AP+yTDnUjmPsXuDfqhuyWRXhfQqo1BWVUQMbzEKIr6iokt62dUckW9aeNWfypD3nBmC8Vbh0j1IZ4IaIUZh/Y4GgLeAyF66UtNz7KjUrX7eDuMdmYIOzoRhCKqoGrOoTYDzqgyUeyG/R6iQ85yPXnCCXaqIDOcL28rFo7z3wr6m8PxXFIm9t+0uPNI+LB6za02VBKJOYPpjVLbcuj/ofSQPZSLnvXCyzWDiB+dxkqoURAW1DlsLULHbZxSqAdPFofvZbpCjSyka/hZ1Q8tECCZ0n2SqznZ3+rqeb2zUV9hMd/Ir9zaMzE0LYIES9Gl/yGqHJKwwT83I2iNbla06pYjGwI6xlJLnhNnLy968+QU6EZ8hgg0knMS7JEARRIfafleaWBqQ3ywq38Ns/jN+Xn22KXdnhDtU09bzO9wZH7MnAjMM2Ru6lhpksPXrb7XhLhv5vmKbrEasKwrCCE/Kop1SSeCInbcFdKJ0RkWSNjgj9s9hYyJi8+40KUMisqKG0Hnkf7EZuog6ECm8/Hx9ELwFHyHzdHklWcOJaswNhoMmOHcBS38ZFQbuwikDzDJUV96n2K3OiZIIrmwPTzKlluW0IhBbJmhi213geAlCHGT8hfUE80BosEh38Z4KJM08yd2cpxyTnGA+XHLPUl0zar8T8w13aaMY/b1K+33k03PX5ryoaiKtup/8opLsXbnF6Yx8ud8l3NXDWi9bLY6yxlgg8ZR9a9A9jt4tqKSrDCd4HoP2RrB8+kcAecty+94KdYz7fJ+DrZlESRiUtlUICke638KGi7xE+2CQHZCtaQlDk4mH4wMm4pftCDl028/SzgT1z5zGvKpfSpJbe3D+gzpJaDXjd3ehCegGW9UmaspVqJXPAV1Syp+IRVgVPlAQENLw8IJe5VpRaQnu2lD6Ey4vb6odt7xqgNZrUt5eZ4a6nqxHkpo/XTEsFHGro4O3hD/CmeNxDFxFDFcw9woc2HXBr1AhGexCD8gZJGnUqKdM10ckimf7Hy2nwI0s4e42moh6HP7UDSGTL+SJwdBpCaICkA3ndCcPqAihDnCjxR1qJ4MA8IlFRJy7NYGJPNo25YeU7enrzbP09rkqKqQHtugC5qB9KjCQH2BcFEdHM02Dx/WXWLfKn0L9U0jHyi7Phj+xJsAq5WDKOs1GraM9rrb4eduH1JvXanFzfR9zxVW0HHfaq8hgvw+gO7hJ+9+CP8fI84Tb2sWqXL/JA1qbCA/CoVHQqtPlVq3lNFF5p9uqs+bZlcu0+VXbglNAwv+J0aFqbwI9e1qJC3/F60LJQD5UA98DNXkCgd5cXdKEpYEzSWX8HA3cy1B3VRf/ldhpsZTbVs9p+hUsDwBmNKcH+hySdx6CLfsGiKicZRKAZO9lPFlPIJGfmV+wIe5FO/J/ajChweP5G76e9scVFkm6UeRA7I7K0gjoPeVqP/cTvN1LZwCxGPIUfcGCwEsIfgA4ME07QaMHrLanQcw6nbHVOtdwyLWSZtNMA2NImnObHbw+WjK7KsqSZi6pQNiZeJx+NjwRFbughT6yw6gxOPAT3BR6uGGimsXrCfPcgA/pZZfemUl0DHbwvHPzjxksQTxOD2J/puUVtEd89NZ2lUNpRjsmT51nGRbAdnr/aKJcM7t8mF90el2hDP8DwRBJB4AgUGu0pSGFwNNj8Htsa3+Ef0m3JMZkc8N9iWRj1LJ/CdcrrupZ1f01e/pVae5mtpPhuHWa0497jISynnhG73Is51Z3XL3ZB9utEdWOHvbfsC1p20c+6iY9zK15Wgr9HyqfIXdO+D1Hx0CJ/6QDGLjQECHOeI9BAhA4SbgxoOqShXsUDg+ViCMWcBiVqiDTIuQ263NzdhRagDx8I4wMqqqkpkpzDo0Z0XRPQwaQyfwD0tNgDfx50vUKqYfP6A0SS9CnYvlsTv86d6FpJFLAfc18LGNqPnBXrFMDNJBnxTQbKgEIeijfGVgocT1/9B3Wiod8GcFmQeSO1k4pwX7gaQG4sP85hs7U3cKMj318kt3U3ajrvRI9/9SzvE/KggQRp0VxFGURa/0lngnmoj8448Qg/7xyEff8M78ih7B+c4r08iv476gt8gv1zuyhRyjWqHZ0cF11FrldRoptjfyKi3yMD4HrfJZQdJIwdKY3tBArRHwnXBJs1o9jFeoDDsjzL5aBMj9/ITqqE89T63B9QZtwfUFW8PqNO3B9Rp2wPq/W8PqI9ve0C92/aAesv2APNdiM8XIejPhahOE/KG9gN/xo2AIxq6rHziuJATnJ4cnx/sv31kGwGoP8T9i1AwOmqR8zv84Jq8X8Q3y7I+kfHTGRqj/KFZXAbdqyume60mG7JZpxpT66ru2Jap1ZlGTctWLNXRV0P36mJ0r66O7otc7xx0XyzZEui+SK6V0b36ddL9FO+85vi5OF4HJ7JTGLLWGD8uAw8la47/0lCVTcOMID/uCBcA+QnuZyFkn2URzkPsXD9rZF/IupbN7OrizK4+TGbXZmR2bcXMrk1ndm0as2v3z+za42N27W7Mrk1i9l2xj8ofd0l88ewswQ8cIXsuSx4Tfe9mR+yoC/iVIHhzfcDvSFxDndwvgdfr5YaaI/gyeFpbLU9rtFZT61rdoLZlwIRa1JZNy2goHdroMKquhqe1xXhaWx1PF7nFOXi6WLIl8HSRXCvjae0r5OmZPOd/GaZ+fEAqPOuaSL80M+TzMCOSjvuSuyLp7St4DEvnwUgxqDVHLmYTywZJbXGQ1B4mSNZnBMn6ikGyPh0k69NAsn7/IFl/fCBZvxtI1ieB5E8uA8dJExIxO8CddfClPQp1fXwb0cwHwF6QkMN98qr19uwxwSfXXnbQm2oQURS5sodPeoMe8XnvwL1f/GyoZV2eeAJ8uA+1cKoKzoBhIpdBq/XV0qrFHIVaTFM0o2bQuqw1OpqpOszRZbXTsFf0bHd9MVqtr45Wi3zvHLRaLNkSaLVIrpXRav0rpNXZ3PN/FVp9ICfAiqbw06eRoLU+/B2XQYSRNWt/aa7K52FG1h73hHdl7dv9z0JHwFNW4TzYLvSzxvbFzGvZ2F5fHNvrDw/bFV3T9BmwnZdbIbbz9m/HdlHkFmzPCtwjtvMuHxe28yHPj+1iciZg+z8S/nlPNsNPFsreVZPuiMLficc/s4t63evPYVDuJVHi9Sjc+fuMRP+Gwe3zk3dnZ62D40f1tu5/AMrroFt8nBv5HPQ7/Ebu/JSZXxV6Ror3uumbv4e0je+YVl/fL/n34bKsTyL/fEbHyB8nfFHw5xa7WvB3WN0C2m8Yqt2xaF1nusEcatfkjlHX6voKwD9dhncF/9HqSwX/Sd57RvCfLNmC4D9JrpWAfzqKrw78F3bw6z2B+fYE6pBr7twMe9M3Be4LxftE1tcg/qVJKZ2FGTC82C/dFcPv7A0WI/Qpa6IIxVFDaxBfxLyWieGjMf5OGD7axIPCcGNGDDdWjOHGdAw3pmG4cf8Ybjw+DDfuhuHGlMewIXK3OqHLfTOYFH4w8aycnTiuTa//0yOvTt792Dr+vvWYWDt/dpu/WzpVIQdvocT7R2dFnoTO2fyMkzOfQi7CK28ZCG2sFqFlo2E4CqWWITtGw6pbYAGMQcZi2TXT0lfwpHe6hBZBaGN1CF3keedA6GLJloDQRXKtDKGNrxChpzvnNSPPw8iKyd+1WRSyHhInK/Kak788yPBZmJGTx53Pgo+GFyz5RUB4JsMvhmFFXsPwIja0bBg2Fodh42HCsDkjDJsrhmFzOgyb02DYvH8YNh8fDJt3g2FzEgy/DlyMyZD1vGeJz6L/+9//V2VGDD6lcQj2RM5ah63r/3H8mBgYtPZUMaKnipk9OC6093RfeWrUnrYalS9AwRMPkHGCsB9z93gMhNNJXAYCm6tFYMPRaUOrmTXDcExm2bLNaMcxbFPVaqqBxrAKBDYXQ2BzdQhc5G/nQOBiyZaAwEVyrQyBza8QgW9zyWv4nfOh8Rokt2OB6kGR7/qE+CFQyxwnxOM+567kW7TSF3w8e4q9TwDe9envQqazbOA1Fwde80ECb702G/DWa6sF3nptKvBCkduBlxe4X+Ct1x4d8NZrdwJemJwJwFv6Cb8XmuK3Qnv8WxdL5NsZgfcHFl5ef8ZvKfhl//Ax4e5TRUGtIb2C3vIvxEIIVvg3TMn3j7uqesv3YcH8jJEuzp7ofgmwW6+t+JO9bMWSbdupaUzRVaVh1B1dt4HxFbNh1mpsJbCLi2YB2B2qvmzYLfS1s8PuBMkWh91CuVYFuziKrw52b3PHa9idD3Yb+I06N4LUQ0JddeJ7o9eoe3+8wmdhNtQt8Dh3Rd2idb4Y6t5u7cWgq6pr0F3EcJYMukNx+a6gO9TEgwJdeUbQlVcMuvJ00JWnga58/6ArPz7Qle8GuvL0x5xB9mjm74XCd46Uhb8DSyCn+8c/PLYviR59wllo7wsArjnxy6DSOSl8PzCfOy6FDvO3DNSVV4u6VDcUXW4A7lp1TeswAz8eTK01LEsB5K2bq0FdeTHUlVeHukXedg7ULZZsCahbJNfKUFf+ClF3kkNeY+5cDzRrZvYGx8Iw9aCA11wD7wPgFnN24B33O4s/1SxW+0LPMs9s8xOw11xj7yLms2zslRfHXvlhYq8yI/YqK8ZeZTr2KtOwV7l/7FUeH/Yqd8NeZRL2vmEkSsA7UnSyPPsZ/oqQKfC7R333+i8I7Lsn796enxw8qs/FfsO/fAl0h1Tr9gTJsuwB5/v/OiaEYG3iKW82Q2MQjHPIBTAW/05kbmkr/k5kvWEzZhm6aZoN3XR0rWFbVGeqWlcc/mGKq8BfZTH8VVaHv0Vedw78LZZsCfhbJNfK8Ff5CvH3dse8huD5INhAICgIVw8Jf7X1ee8D4BdtjvPecb9zV/wtXu2LQfB0my/GX2196ruQ+Swbf5XF8Vd5mPirzoi/6orxV52Ov+o0/FXvH3/Vx4e/6t3wV52Ev0cU5swHNYBbmfV9vIAybh+SpPP9H39sPaoHm4/wOeZcX18Abyd/6DOfi/G366ZztQywVVf8fl2Lyg0DGVZxVJhIQ5Frikk1TVMcp15fzUdW1dXFwFZdHdgW+dM5wLZYsiWAbZFcKwNb9SsE2yKXu8bZOT+kCr9WZiwQPSiYXb9P9yHQyOzv0y3wNXeF2eEVvuBnUk2x8wkAu35/7kIms2yAVRcHWPVhAqw2I8BqKwZYbTrAatMAVrt/gNUeH8BqdwNYbRLAHvJ3h0AIjhjpCecCkbmH0zor0L4Orz+D8kPXJif/ah0f7z8mpD3M35pbg6r8G47xe40zlOWPNAtt3j/u1id+ULOYpzHcfR2Kzp30e5oQ4RYHX2214FunjuPIDV2znY5j6p1GR1dsu2Gxjq0aWoeuBny1xcBXWx34FvnhOcC3WLIlgG+RXCsDX+0rBN9ZXPUahOcCYQM/qXSnIIA9JBSurz+s+QFwTX32D2su8D53ReHb1vxCaDyD5RfDcX39ac0LGdGy4VhbHI61hwnHjRnhuLFiOG5Mh+PGNDhu3D8cNx4fHDfuBseNSXA84ZvdS+lXu5dmBOQb39/+eOCY648TLgdjrsPsAece5R9lpYAu8w+3un9Ebkx84PlwH27jhI1RMp/OZYBxY8UfamXQRoM5jskU2qkzGxa7DuyraLquGXWmrAaMG4uBcWN1YFzkg+cA42LJlgDGRXKtDIwbXyEYz+qm13A856c54+nZjeD1kMC4sX7g+QEwTWOOB57Hvc9dwXjaml/wc51vtfxiMG6sH3teyIiWDcaNxcG48TDBWJ8RjPUVg7E+HYz1aWCs3z8Y648PjPW7gbE+8dSYEVAECO+4FJRBPXS/R+ioZ/7cK2r/jtH1+OSf+28fExMfIs9y5SEJp+rLoFio8P4xWJ94UsznZ/yTr8TsLQOC9RVDsKnohl6vObZsMrVuKx2nLmsNRVZ1pdNZ1cdd6YtBsL46CC7yt3NAcLFkS4DgIrlWBsH6VwjBM7jkNf/O+ZQ0ZtU3AtVD4l99fTD8ANBFn+NgeNzx3PlgePJyX/CR6VuNvhh99fWZ8EL2s2z01RdHX/1hoq8xI/oaK0ZfYzr6GtPQ17h/9DUeH/oad0NfYzL6RiQEDwI+17cxPJLEh+H0AnC8W+R7aoUu89ATgwOd5zOgoYbLyOnhybtfHhcO44lw+txxptT8YeoEz4aFcvNrqYoFLYOS75eWT5GWGxM/Khqnr/hzovn8LgOYjdUCc0d1bL2jMqWh03rNoErNsWzNaSgdVavRmr4aYDYWA2ZjdcBc5KXnAOZiyZYAzEVyrQyYja8SmOd35GuCnouga4PP0R0Kb2RTNlW9XIH/nj8koD4lemMN1F8aiNJZmBGoxx3T3YF6dnewEGHPtyiKgBtVtAbuRexr2cBtLA7cxsMEbnNG4DZXDNzmdOA2pwG3ef/AbT4+4DbvBtzmROCW9iOQOsbsyJNofHH92Y16rNzyHfDXZPfk6HyfnJ2eHP90cHi4PyNyH4Ne4SZ5u3/cerv3qID7qaILhWaHzoDSOqhV8HGuWvEnAjQqGH7nKr7/42lj4vG0mL8x4k6ndxm4ba74fJp11HrHkU2T2bZp2A3V0TqmxkzbUmAFGqvBbXMx3DZXh9tFPnoO3C6WbAm4XSTXynDb/Bpx+05ufA3ccwB3XTUwU74Z2x4OYfeJsT6y/uIElM7CjIQ97ovuTNhzeYAFGHvqOig+xTbWp9gLmdSyodpcHKrNWaAa5gqj6+9JEOOCuFBTtWCTMcY5AErWA8W5Ifqn/cEfuUKgzkaakdnM8/rUcVy/C2AzNIRBE6NZ24ZI2y5GBpvCB39tW9LObuCwskVD3ml8cXt5GAfWiNltRWMIfG3PtUBcKH2W9PtBGN9WAXREkXcHlQ7hiudG/OptNSNmj1Y8ExeyOjt7YJeB71owQzGPsRdjEYJdMl/EBljUqPWm1JB2aGInPhnodmBuG7kdUoHmEozXT3rQBkzWXhr0IwKX8PMPfufLGqce87l0JkXPeT2RXd+4nM2lqmICbkObLByO16hlKMfXIuoEXiPHNMAyOMYftnZhrbVxfZEyof0+pExcodW+08khcVop4A8GjbUtiKIfpME+Aghw6XZXL0C6oSHVJK5AscbzdflHEPTaYhugLdTWGHzzgai6RZT+RxIFnuuQsGttGvUXRPyDtNui9oduGCQQYmDyAyiMRZQ63M9+QKl+ELko0RahFjSEGxzkjzL3hNB6rVbbJlkEI2kIE3ICB6IPyKQekKAADUoGyBD4vSCJWJDEzdIfzSx1BOCO9z2GL199OnA2iwcMIv5R4YOupHI0JZRD2ialvGHcCVhKy9jq8DykVgvW4EMsYe3ULjEGoLGMWnVbIgMfJZELhnmYeJ1aVbo7xTevQtyVGnbIT3IbKIin3KDa2aIasqvMmQ5Jm9krbjlBiukEFiMtOwwsGpPTvdeZ+85WXZrt3bZCUyias+Rw/BKdom2k3X9bLpOO65O9g58IhEq/jZtM7U4A/jck5fJOvp8gLsGsbOTxJb3WDqMIZ2sj26AYK+C5/od2lPT5nGbupXQz0OM+EU/l+YYIfwVt6KWdI9AQLI2IeNefu9RDXwcxXAwx73RCr1fMgoXFsGfoON0GSQtYLiBe0EYHW0pluojj/la1Ku7EFww8a6XnRhF077BOiIcrlU6Y74keiVuYE77mN0HYm5cKhH0ySdp+zwJJMw2lwlxdXVW6QdD1sOvMWwpnufMa4wZJfAZ0b18w/sE69JLZ5HteYbjvJ0MK8FzmY2fpRN8YPXYYuV2rAostH2qlUsFuuG/C8UE4hJ77QRISjxIIQ46LSfFfqLFKpTTm1aE3Lk1uJtxDpRk82Ri1yGpqdWiRaIX8dmaJrh+HAapp5IKYTeE1MEgWzAPmOaMepdSXSzdzsewOfteg2IcqgQdL9hj4cBhKXNr5ZoOQl567U2TJpZ2WbSfM9fKtlbHSfyshLEMuRDIz41rjVo2FoTj0m+0/g2MF+09K2S3RUqrdktDu8PxFN80X/Qt0xi7hR8UOeqWdXX6FnF3/dRmKbnNBZ+wA1qldce0+GCR/Xa04MCMHfgTGksRkFywBgiGeNYL6T2no3qkXK6kktvdppk4OP2FGNtoHbmFx7d+Yhsm9psZfsOKL13XxBFfx2WbMLoas4ntxaagSF2/I9rJXBcuAGziuA75mcru3KLRPk7ZSGiWC4ar4OdSwiIQz3xD1bz+Bc30XonNIu5vPt59Uq7lfCb/9Nj9HI1xOK3A+CYkv4h7fLfxm45uN/w9tk+kGFysBAA=="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514831.13/warc/CC-MAIN-20240718130417-20240718160417-00254.warc.gz", "offset": 3241425, "length": 6870, "data": "H4sIAAAAAAAAAO0963bbNtK/rXPyDoh2Y9pfrAt1syxZ2mPLauKsb7WdttueHh6IhCTEFMkSpC/tyfNs8xx+mu8tvhmApEiJcpw62e7uF7eWKVxmBoPB3AAi3++dDyp6ufqs8D08lS7vPNYhPhOe6wgWFR7QAApr1VqjVN0u6e1Lvd6p651m68eowTkzXd8qHR50yG7oO50w5FanXtset+l4uzTWa3qpMWatEq024WmnVm83qw2Tmc3+s8LAdQLmBKUj5kyCKaBpNnda82JFEPU8m5s04K5TmQaB1yUzMQmgqrdA6vfUN7kzdhdoqbV2aNtqtEqs3TCRAr3Uro1GJX1n1Go2GjW206r3IxCA2Qx9XyJ300AsWhvrdDwusVptDAOqNku0sV0tmc3WqDmqb7faOouBHJ6V9iwLiBMdold3yrV6o6y3WuVawmjqT1hQent+2CE4ok6lMuIjm7vBlP0SsvKMCwGjtdjYp47JymO/wuHbbdmben+zr+0eDYOp6xuCsXVu9er6TjOCfOa7gWu6toILc6vHFfTOdqlVOuATJoIOEVMKk3h8dPz64Nv9N6/3L5pvTl/VB9/+4/yo8ffLxmn9rHH0Kuq7b7vmVbbn/v728PC708H+SePk2736fqN+9P2P/zg9Gh60BoP9b4cxJyzgJB9zZiUELM/p7TSY2S9vZ/azwrPC68vLMyQbJK5KTv/+rKDk73IabhG9Td6EtpRFEklhk7w6vlwUmIDdBhUE2iXmlPqCBb0wGJfazwo/lAY+vbGZX7oEzooxPAwd07W4M+lA29C5YpaE5jATaeuQK8a8ErX5NQjZd9S/65A902RekPSD5tScMpScwEfGe64ISlBiXvWqW8TzWerLqqpnheGtx0FiOuSCBlukqpM31EEeVEm12pH/q5FegNwMXPeKwzjPpt7x3T4KTun0bG9wsN8zqcla9WrTSItTlzAFvHfsOlukXi29CZ0ScLEJHO00Yy52yTG9Le1NWK9eVT8L2F6fXQwvLg4Penpje9zcNmE5bVd36rSxA2t9e8zqY6veaDdHZpd4NJj2Kgv9Z6NTj5qlo9NXhyc9i9ksYNacNjnDVb0E44aVuZ2MW8/StkhUBBQpO9k7HvaigtWtgP5Wu7rd1lt6vZbf7GDvcm9/72L4FCKPKEz1MUgICv9H5feC+dfM7xC3Jm54YE5LZ+4N8/ddlJfrelpuY0mfi+3Iz9Gjje3GDi6nwu7zg9PB5T/OhgQXBDl7u390OCDFUqXyfX1QqRxcHpAfXl8eHxFYc5XK8KRYIPBTjLTSzc1N+aZedv1J5fJcLVRdx07xc9kKrGK/sCuBwxJ2RC+nr76zs6N6FLFRx6bOpFcc+8VkfWpygWoIiVGrX1jbnbGASiVWAhnm172iGY0RNT90VN96xVWLvUgqCRiHzlivqJRmquvZ8T555buhJ9tmG1+xuxswayLVPNLKxGJE6eUtkl5mywgtJkyfe6hIUmD2oz7rbOLTa9aFruR4DvobCTqHIt8duUGaHmrbEc7npdLhkLCACIDBHFBrCMl0Z7AO+YjbPPjfUimPqdoPpbd7IFOqoc20BLp2OOwNrQnTFsZ0zdmN5/pBio4bbsFqt9g1N1lJftki3OEBp3ZJmNRmPX2LzOgtn4WzuABkZm1tDUAHPLBZf0ADarsT4ARziM0nTg5LdiuqbWEN/9u1uXMFvord00RwZzMxZSzQiHQLNCkTphAamfps3NPKFdWmAiyZgcVRf8rQ4m/6dq3a2qk1WlVN8fIpcMfAkhK9YcKdsc8P3fWYUxJgtj4/aF/6cIbFhWfTu88I35uNGsbMGgMiG61qlvbt9meBPYGVwHzODG8K3s8XQRE/fBHYPneCLwE4oJMvwoxrjivTYaFg+eBBHxGoMFDjogOoHD0COiiN+uL16fnl4O0lORycnsTY+IyCm1kRPGCVMQWVAusUPqQaUtpUkan0/jt6TVVpkQjf7BW5Y9oh6N1UTcXy6cRwDMt3vfI7UezvVlTFE0BOqWPZ7LOC9FwvfBiYtgAMeULIGh+TjeeWa4YzYHUZ4oqhzfBR7N8NbCrECejtzd9IpQLuZ+gTzlDzfqQ96RFsNQ4d6QZvOO7MMLFy8zcsX7umAChgM9Fz2A3Z8316t7HZTapMNwTDUFUFY9cnG7IDlBC+m4/6kk4Q8Yb2P9pm2ZZuDDR++XKT/EYkLWtyoI/ojHRt8M2yOR9Mj8wHQNQIABzS/5Mk9eXLn2HEj4fdVRShp/RePak/PgsgYFSckYOX5fgL3P/GVcwklIQB2GTBwUzLKbE1ht4cCD5YO0Henh8JtIPzmS7Mp4LIpsyARhsWWE01IcheIH8ADhA4tOmxoE9EzYDJ8r+tKO9ky8GHktTz8UYaaI8U315+A55VJATRaBVBEMuiG+E6AEQRJkG8Z7Zg2ebgAXgs3USxaC71hU9UT7lr3n3nViz+jgcVUEEzqAxCBxRB9KcsAYCu+ugCU5x954KHPeYT4CwOxkMenTpHEM9CtOmHbAsKIUgG16ZDtLFfGvsaFnFxwEYh+OhjCmzAElCZZzblzhuBwWXcFYKOK1R7HfKT5BW6WgAIFO4+FUyCkuBlSKqVyxX5v4aM+xkrLeZhXw2iaqmmK8dMCAR4Ebg+02QbIM4eAZ5OsqgzjdSc3nDHcm/KAOIAxoyjfS/nKCooz6IuUIMLPwNh47fQt4G6Cn1HVaZi5lohOHv4fR2IZ5Ne3F/bIhYEVGZwCH5iR3LnvZQGHBJQC1jn8vDwHEmN+qAUxB+gXDHAiOF+DrDAlwTyYwnuo0i5MOWGmLo3hnBNcJQNhwUQb1wR0JGP1v+KxjwKBZ+BhjEEo745XaDuUUDzB46Rg+FjsCOCRajAWBW6FXZHrnVHXAfTPRgdSJGy2JiGdnAR0CAUPQ1DL5lKuf9dEC+EiMjUukVCOPQApiJ7io9YnYlWHAOWExc8DRZp7g1uSZFGZod2zwltuyv1sEmF1LKObC2ItAoch1SI9ehMQco3Cft3h9aGxmztJbdeaoMpty1tU4FWIGMvuiAV6PME3ObjICsgCqRUwHH3RNGHtpAAoooV5iq0FYyM/SVogKF/ZGGJMrFKPwMyqPmJ/4wA94LA56MwAEAcxofKX3Odic0CYXAxsoxozl5GbE5IA8oUFGX810Y+o1fq+X2i7UliFbCTzR/ih8IqkUpssWV6bvNN8ujuKXJjAHF/SbPNy6DUodcJ2LJuZLQjixWi5LxPCZtctzdTGmz8Erocgl1LeRUqdyMLFVNA1hjRhj+cHRlHpwONdCQfVtJq8WuD3Xq2AbpeIy8RblkawHIkUkCoNsJkrNZ9HKSVUBww1FpX6tsHwETMS0NKeVVES0mCwR1wJ/j1p0FNj3Ql5BhuIStPc+Z+HsYqljyRr4+anU9ha2bwn42l88laYGqkpiOGoiT3CH6WA/cIU5IDYHra05eLJ1f5dguxzx7aidOdxA1E6okVqsvmkerKUV6yd576kmta+fQfU2GG9hLHlNJdcTQAPR83Ex8TkjmGh2Uk4yEvjGFOyfM56RjTa2R9nTzcDCLnxzQT4WTyiGYjKq7AA59z6xHsSumDpNcqlj0wbWWfAd9MFsu4toX81TYfXsCxtZn/fb8WqaX3WYl/j6r9szhHAYVxC2A9ur+f4hA+AuhngzfjwlzyB4Gh6HhpNMDOyNae9mvpEHccO3q1WsXW0CbVVCbjIYxhfrGflM2gYP5Vw68GuoTMBwDTej9OfN//jilvixGb5mV5oaVClwFsTF1fGGPXDRBnNLdPYASmpow455JiB8CM0XLc0DOikCUamBR3cFFrxZjIDKmaAwtvQiEcwkhDQH9n0t8j4FoEXOXkWRAw+A6h/rUbCoz/r9mvpAP4VePCLk45IyqprgH7XwB1gQ+/VoRdIZH6yHWAJ9wHqQPyGXM00t+lKjAulhf2jOW3YhYGPAeRIwxYhEed/jkLXPB6mE9oiFuUM0xT3H8wYXYIvWYmgXgcRuI7/P53f+5Hl8tlGAMC2K1Q+A0sDLWQbinDLGYUpgbHzJ+hayWF4C8JyzA/mHBfA9igKwyACARoMdls5gV3WhrYXw8hBp/QEfV5UMZ9oK1ropfr5Vq5TmpVvYWnFqrbpLrdabY7zW3yzgPs1HEYGd565K8SKU4VSH06Z2cKAWJWBjGJ/XlDTQzowKgA4ncs6RbeF7INffdmuVkJSpebmsy2c9piMWgnj1q4x1fy+WQadEjTu0UIuyohi2THoipBMliRtpCLA5ebms6D+w9YLJebBs1Y6CfzBI1SSzoFw0gtb4SWbiIYi4Uo+S7XYyS4UZX8YmQaQAs/BgOrmQd3C3hkEyte3GmsUfPMTGSoyNb0E4uTXrQKjs2umV1LNVlT7ILve5I9a2tH9x9AJQ1eDy8uT78jccOIX+l+j/5ZICMjKXJdwiwjM/BkAcxgGsn8MVbCyzWPQRNDN3Bb0BXMeiyahZ9CtkH6i1r10ZOvhCJa/Qg/0zSxOSmpQ0snNxzXUhKcGU+6oRHwwIe5PoicCtBKHI8D4T6mkBqLQ5mH6WgWkBzRz6gaO6AAzZzGRGRwg19NHWvPtouJetXmtqSTVEvbv6FvdlVmJ52EMahtb0QV0k2/drkAs5apBbR8Nol1HTwanh0K/J3bMrU/olBit/KEjzUycn3QyT2tqskRqWqo1SJ1rIadGRN8Mdm5HPqFJKLYX3dGwuvmtTZd26aeYCt5kGqwsbk8jpnLHSE/FkcSd1w5lrjBU0fzRz0FwWxmLqW7lBRnv8rwBkXAmKBv7fqB3B5JZ80wdopbJ/mEuLURbalvWMyWBZhaSO1YRCGXjzHXw3lW7LwuwlEvBr3u018VRdrLDeSBO06wbEpXP8TjZqCKLQ23CaIq0iGatvlSA1ie5xtATaq/pG2pK5QmvdDPMZgD4wp90QMn5EX9YH507UWtxS0oweNrc799jWRZWI45B+Peqm5p2pa+FSVO5xyDiupmyvVfYvFSj4347GCavStDurEPEY4R5ffAKcJTELgdBG6Ej6dnelmS8QFFbGNOk5w6DN4eSlatQJMfJUeiG6Mg6keGzMl2G9+Ve8DRflo6WF5Lkv3smtob2AwDMKR6ASJJQqn5Q8LXK24rSTEk7RuLwiqrcBi9Txz0nHGYQkjAbM7JT8pSmTsQlpl7zWRWNtVpKShcXsgpXQLx54Vc864vdZ3rDCCgveppUojAnjIfJnZ51YL+JtQOesVLn2eMUZHIkyt5FVG0ZcL6AAcYDw9C1MH82BJkVaXUi6M7g/6qFCXFkzI9beQGgTvTyFTqwJ5WX1KhkkzcSs8qUJCUmTrWo81MA8QA1hlAlTOrtGE6fpi5PjOUqRRJmBj7rTL2JuDVT13Ah+ccMQDijhfGKnfKLQsDFIVPmb2fftaIRAsFRkOr9B/oIfliVA1sGHd6uIfrLTQffvtwhzFntqX6pCijnVrnN9HRtzvrv4Ru0BXM58D5X5kVGW9V3BWdmh41oR0du7SirxfDvfPB66jZSfd90kHXE5jSC0Ci5tCq2UpwJzxwqSWRImr1Xqs8yGZUwMlAdNm4grP+WOO52iGgY9T4voE+E4MeLOUWpCUnNTIDs0hWDzQMg+qEezPczgZwPfg1lCnTogWj7SksmClIEGn9vNK0XMsEGzIIY6SUnV69j4Q/4KhGWkgsu24/PVgbKdife41Wq9YtLGmYQtrlLTIbm51JpZUEMQp2SamyOF5KMzxGbjHP5lRpj1gzRQ4qbhbj1hSC17bk7jLoqWVfdEOTLVZ6o0k9iXZA5AZttyjPm6CJyfNSlQ9VLFdAL4J2kpPvuDN47GH1uivPQTrM7unF6Cwhnw1viymWHM4miaq0VNxajFVZsVqMFVyxjsceE/1YTPQjajiDGnrrVm+hhixKdazFAYI6GM4IrCOfz9YZNSEm6Cay9pFmlZVzUsIkFy0SPGc0wZnBCvVVJRRUMzVSdRgJCgwccTG7qATujmJCBliUdXOjNEtPzQyJma/Sa4YcAawLzK05EPxAvPPm/oMfCgqGNmf1xv2ULwaOLamQTNCb1+fH+k67nXDLDG6Na5gXKC2326VatdpYpzOv648DtCTGeBb08F2MF/U9d8xf1L6BAni+YtfwOQtu4XPkuldxH/AKHJ/1MkUjhQpG9WJQf7HXhpG9qFVZAB9vZMlONMKkx6d3QAUpYNk2oY2XFFvgRPf0nXp7HYcD05bUeOBTvylD4+987sxLMUPdgxXNRVJGQ0yu9SKGpoplvq53pAhyncUOF4PXR6fDH4fnS132XYAPqC2mJcKR+ZMsz5FPYnGdx9wL+kf6SIvqx1SFyrgXZ9SfcKek/ItOy7vtps09btdn1ApaGBK4VwycB2o1xu06HYFctKvtnfHOeFtvN+Bnu2rVLAZ+CfI44Gik9O2aXq/uNPUmZjpXZj9lJiq0pUOTsxWtFkYkq3lb1eq03BqeQEzByOuds+szN25/iZdA8Uy2QC1sKi083xfW1Dl7UMKrVKnqrKyWzaMZylKGZKykK9leyaPs8GL/IJ8urHmAKqzO0lRQccCaSuaEtsrqzPNMmu/ezPcHUokd3AVL83bBYZQ2uhtPOM5xDDBI9BmmvCMfV2ZAE0aMJqWJz+6yWRoNxOkXw5zCstH6l5gbkml1tT5AfFCEMu0jTj6oPJUD1CFv8eCaoBzzqB4sEle48IfhITU+4XgSNoNKJr6zQ7rzIAJ62pjAmCD+OJKKaVscZNp83X9YRZBKiIknUaQSphjvLzE6zxfMf3NNI/0F20P7W0SBJl3yeEgtgCR1JDIpUaMK3KVPrdBUKcB8fjBLfxIvhhaXFC/L3KoBSPkT4L7GY2huwwikFQGOvikTtDHJbkoOyXegdp82f45z/0HKlFoL8oDhKrFCe7iKEkfa0SfRcjjD10pwCzBHnMBAE28VcgGx2JNQfwMREQ1y0Opt0iTmbBVifI0pfOKwjxSMZbHB7dD7f1KQhY/2BZkHO03tFBhyHgrUSfmEyyThk8ge0OD+wwQQM7FKXlbKvcQ+X7cNFPrXh0enF6fwZ0gOhuRoj5wPjw5fHZ6eSPmX/syqwUT7J8HTpE9SSCDqMCGYsNknDyqmIh5XrQo2r9+gRzgANYX94Q+HF5fDk8vDvaPDi+MhKZHj04Ph+Un0ZXiLWw8OvjeFD4L4OIdi2bBEeyqF2OpKny9td+eeQ06apvt0W0d+ytqYnyF++PfV+GpcZfWnFKU0/oBSjvmT0sxbREYJOQikxlKPXRLpkXIkyB9Zz9GEx+j+kCJYgKG0QZ6r9VUV/DeogvgsypKKkJmtwnwzFTWE3Dc12C2bgU7A15rBeZl/yW6RRnv6zLajswg9bR6PpEFkQ7aCitmm85aeHZ0FUvkmcwSC41qsNKK+RBpMH27v4qbsAD4fahqAa2zYfATkQuuL0EOX4qEO8UsV805H+BYHF7L0oZ5CXRCQwqYK4j79gyjNj2/fSi98uiSx7Jo5WvpgT8KaxnajEQtatlLyQcpStjY79CN+7bPFJjmDZeSMAf/DWyJA9R5T31nqtTTQYzqhAtVfDnmCB6HEASIFtiF0pPzEak+dcDpI9j+SQ09kLrmFtOgmRi1nxzwTGS/X58ak2cRtBkKmKjlglMqe/OtSyO3HpZDbXzaF3P5oCrn9kRRy+1+eQm7/v0sht/9QCrm9MoX8d878Kzah1LfQObRpxgNksVlCTfCfnldODTXOEtsU08DzAWN5esgLieYnQVCZ53q7kZt5bv4JmefzvVfD00vAcFn+c5LS7S+clLaazXZN18fjVsNs661G26w39O12vcl0NqqNrC+RlG4/KSnd/mJJ6Vzl/Oik9Aq6npyUzqXqCyWl2/8BSek/oI6/5p//nPwzKBaApHQouSwvJiG+ZrL/rTLZzT8rkw0G/0/JZO98zWN/0eRVo57NXf2nJqv09o5MVu0/PlmVMlIfMUIe84UrD7A9adAn7ox4vuv5K4ebtpsX6244s7sQFH96Cj3HR4hS6J9gmf89suX/evv0pfPuzTwEUr8mefedr1n3r4rrv1FxfU34/7cl/FsPJ/z3vyb801mHP5jwbz8q4b9bSZ8gT7/bPZJhylSOzmT4RsLHLynCt5RTl8cETAQGnsHfwI/NwtpveEYdX3LB72UMP8rywD7pEx1fd5Ev1ICG9ION4kkI/oPvyjgH3yEHbnrk/oPNru8/kOfF5MKfCM4YInURvYSUSaoUopek8tDuPhLtiIpPRpmU4caCvFVHvvOdXMmVen9VcRsvgUq9poKPq99QWfadQM3Cmsf7ZEWvtm5HWZAeXXhVhbiOCEczHuC7oIq+7CzJl1mW3seRGVCp8+P3cQJ8icSPXn5Zao8vNy02ly88uSHeTCZZq1f0vK4OiNViVyzLR4Q528XWWKYl73/Eju9uJRLj5HuyhuTTmtp7ytTO3+VVb/NzhxwcfkcWb2nAN+uT9G5ycUMhMUtRmeHLN+3x/jCVL15qgLcBGiL01Lvoq683GLsydpcpavkEMLaL/WP0w2GCiQ0Ok/LSwEFQBjhBugLrDRvhTaSIGRBHkhk1UHc/GyiYxYimx93ln2xQLd/63M+5D2OJ2LVV1HqzEa6XiEOpa7gnrjuxETUJ5L880DNGEPNf9b9BU05Ch83fo1F3PLySHdK411IMsDlzEFl8HUd29IhQ8Mmo7LAgGWq5XEY0wrVDqQXBuABmdfslJaBdLE6jm0GgaTGmU1OEan3AJqlJxEQmgSOBJIWsRFYiqUOJVPc7pDYa5K0exf7iNR9yzqKLDMARyr+XZCHNXPT04qLViWtwX07tCxRnzAkPmO+GMJSg2H9WIGTX5rkXdRT7e6YZMm4n+e6l1n8pYtoTb94Ype9TkVL9TF7ejnjj3UDQLCD/YTGuUpAi7hYVd9PzJxbFF9UDIGPgXoiy6c6K/YEsIRf3v1/7Cm1C6CMR4O1+ZW56+O9a4HOlbMGMHDoChCUMCIQ2U9fm0SUxMgT8I1hGYTk07btHITm6c51FHLivILm/MA2rsUbCn7Pi89d1/gRX8J/ZwH8XIyUVr1RRqpMkb1Fb5y8DKeDy8uW19EIYUYBPQ2PxKpt01+R+DrmMVP+H3R28+N3AHWrwAdYqlUSv+M+fJ2aeSDrxlkhFMd7agk/4Dxb8HwFw+UOTZgAA"} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514659.22/warc/CC-MAIN-20240715040934-20240715070934-00517.warc.gz", "offset": 3365888, "length": 10778, "data": "H4sIAAAAAAAAAO19+3LbxtLn32KV32HC89mQ1iYJgHdK1ClZomM5sqTokpyTVIo1AIYkYhCgcZGspPwA50m+uL6qrX2G1dPsW2z3DACCJEhR1zAxfJFIoGe6Z6anp3/dg8GPOye7JaUoP8v9CJ8KZ1cj1iIu80aO7bHw4h714aIqq5WCXC8o1TO52pJrrWr9p5DghOmOaxT291pkK3DtVhCYRsuoyLRWlllBqfeahUqlWik0dK1e0OqGVqFauaxWlO1nuV3H9pntFw6Y3fcHLdJQmrX6+LIQiI5GlqlT33Ts0sD3R5tk6PV9uNWeEvVH6uqm3XOmZKn0elTTWKVglCsoS0MvNBSmFSp6XdXVSrPM9PJ2WAVw1gPX5cydZCVKtSwbrNYrsEaTFiqyIheaZUUpaLVmr2lUaR3+RZXsHxd2DAOE81pEkZtFtVwpKrVaUY07mrp95hfOT/ZbBFvUKpU0U7NMxx+wjwErDk3Pg9YarOdSW2fFnlsy4dun4mgw+qd1YbVp4A8ct+sx9sI02uW44mPX8R3dsUS1MLRKdINeWQ41Cntmn3l+i3gDqrQOfvru+OD7avn45M3h9++Of/hx/7vjw39XDt7tf/f29W4nGuDXlqN/mCz5773D1z90znfPyyfln87qO4f13Tf7u2c/7u/X3p5/t/9j1BEGdKTZM5kRCzA7pJ8G/tB6+WloPcs9y709OztGsUHhZHL03bOcUL/3jv2KKFXyLrC4KpJQCWvk2/dn0/ris09+CSvdJPqAuh7z24HfKzSe5f5V2HXppcXcwhl0rNeDDx1bdwzT7reANrA/MIPXZjMdZWuRD4yNCtQyL0DHfqDuVYvs6Dob+XE5IKf6gKHi+C52/Mjx/AJc0T+05Vdk5LLEl3m3nuU6n0YmKEyLnFL/FZEV8o7a2AcykeUW/ydaegpqs+s4H0xo5/Fg9P7qNepN4eh4Z3fvdVunOoNZV+0mtWmTMFF5+41rviJqvfAusAvQi1VQzpYa9eImeU8/FXb6rF2WxZ8pbm+PTzunp/t7bbncbOj1iq43gKisUkOvNutMM5o6fK5q8iYZUX/QLk2VH2pHI6oXDo6+3T9sG8xiPjPGsp0NAmx3AdpdUJr1uN3KpGzTQoWVomSHO+877fDCfCqQH6a8Ktdqar2RTra3c7bzeue0cx8hDygM9XvQEFT+G/X3lLkXzG0RR/UuTV8fFI6dS+a+dlBfLspJvY00fay2mptmRhU0Cc9yua1v9o52z/593CE4Icjx+euD/V2SL5RKP5Z3S6W9sz3yr7dn7w8IzLlSqXOYzxH4kw+N0uXlZfGyXHTcfunsRExURcFC0eei4Rv57dwWrxymsO21U8oqzWZTlMgjUcuidr+d77n5eH5KfIJKWBOjxnZubWvIfMqNWAF02Lxo5/WwjWj4oaD41s7Pm+x5UoqrsemQtfPCZiaKHr9/Tb51nWDEaSeJP7CrS1jVvAR5aJSJwYgwy69IcprNMjSYp7vmCA1JoprXYZkXrO/SC7YJRcn7cdVveNUpErmO5vhJeahlhTy/KRT2O4T5xIM6mA1mDWvSnSHMQ1MzLdP/f4VCWqdK/yqc74BOCUKLSXHt0n6n3TH6TJpq04XJLkeO6yfkuDQNmO0GuzB1VuBfXhHTNn2TWgVPpxZrK6/IkH4yh8EwugA6s7a2BlX7pm+x7V3qU8vpQ08wm1hm307pkq2SoM2t4d8ty7Q/gKtitSXPv7KYN2DMlwj3CiSuE7rnSWTgsl5bKpYETQm6ZAgrjvhVBIp/KnUwBU21UpMl0Zf3qbcHXVKgl8xzhuzha3dGzC54sGw9fNUud+G6humNLHr1gPWPhlqlOzR6wMjCVXVS9nrjQeruw0xgrsm6owF4P4/CIvrwKHW7pu0/RsU+7T9KZ1yYODNtFngsvXqwRwRudNHiogMoHD0CNijJ+vTt0cnZ7vkZ2d89Ooy4mUMKbmbJM31W6lEwKTBP4Qc3Q8KaCjGF3f+VXlBxNU88V2/nTVu3ArC7iTslw6X9rt01XGdU/NXLb2+VxI17VDmgtmGxB61y5IyCxZVJU5VhnxCyZvbI+jeGowdD6OoiwIqOxfCj9/pq16Kedwh2e+N3UiqB+xm4xGRoeW+gJ22CVL3A5m7wuu0Muzre3Pgdr69dUKjIZ0OvbbNLsuO69Gp9YzO+pTsBLAyyuNBzXLLOC8AVYm6lsz6jfWS8Lv0vaaNocTcGiF++3CC/Ey7LGm/oEoVRrnVzo6iPG9Mm4wYQ0QKoDuX/mYv68uUv0OLl694UEqGn9Fl8Er9c5gNeFD3DG8+v43/o/TeO6ExCSeDDmuyZsEzzIbEkht4cKD6sdh45PznwcB0cj3RuPBSEk7IuEK0bsGqKAcHuBfF3wQEChzbZFvSJqO4zfv2fc663Jq+DD8WlN3vryUrbJH9+9gY8q1AJwtYKgQDKohvh2FCJEIxX8ZlZHpskBw9gxJIkoovGWp+7pXlKnfPOr07JMH81/RKYoCHc9AMbDEH4q8grAFt14wQTPfurAx52z+xDz2JjRthHR/YB4FlAm27AXsFFAMng2rSI1HMLPVfCS6a3x7QAfPQehW7AK2Ayjy1q2u88BJdRUQAdH9DstcjPvK/Q1YKKwOC+ph7jVfHqOSSVisUS/ydhx/2CNw02wrISoGpupkvvmedhhae+4zKJ04BwlgZ8WvGkniASY3pp2oZzWYQq9qDN2NrPfIzCC8VhWATu4MSfqGH998C1QLoS/ZWKQMXQMQJw9vD7CxCe9dtReekVMQBQ6f4++Ikt3jufuTZgk0Ba4DrWh8VjxC3qQi2IfoBxRYAR1fsQ1UK/xDUvK/A2qpQDQ971Bs5l13N0cJS7NvMBb3wgYCOXtv9CxjQJPXMIFqbrMerqgynplqo0veGIHLough3Pn64VOlZAt9yW5hhXxLEx3IPogKuUwXo0sPxTn/qB15YQevFQyvUfHhkFgIh0aTNPiAkloFOxe/JLzM7YKvaAy6EDngYLLfe6aXCVxs4OrLYdWNYmt8M69biVtTm1R/iqYGKTcpEdHYqa0peE11f7xrrELOmlabyUdgemZUgbompRZeRF57gB/SaubmO5mkUlokpugKPisaEPLI9XEN6Ys1wFlqhjYv0luABD+XCFJWKJFfYZmMGdn81fsMId33dNLfChIhPah8Zfcuy+xXyva3qa0Q3H7GXYzbFoIJmoRSz+a5rL6Afx+XNs7Um8KmAhy1zUH4IrZ8q5RSvTN5a5QZYunhA3qiAqz2W2zCIYdSh1CGvZZrhohytWgJrzOaFsfN5eDqi//jFwTAC7hvAqROyGXxSdArrGiNT51/FB9+BoVyIt3g9zZTXMiy77NLK6YOsl8hLrLfIFsBiqFAgqaRiMlTaXq2luLTYs1NImt7cLqgk7L1lTwqsiUkITuqYN7oR5cbtaky2dW3NUb25Sn8ad+zAdK7rknv261OjcplsnGv9gXToerKlODc102KGoyW2CP4u+c4AhyV3o9KSnzydPqvHdzEU+e2DFTneMGwi3E3NMl2WGpivFePHSaeaLz2nh099kwrrSS2xTwnZFaABKLjcSNynJmMNiHZnwkKfaMJbkm7HoiOkl8uIFWUwGyHkZMi/o95cg06j3ATzwcW8t0V0JexCXmtdlC4at6DLoN51FOi69wv6VNhZP4Gi1Gf/+vBaapc+TGv8ZTfuDOEc+hXZ70PXo/t7GIVyi0gerb2h6+ow/CB2KjpdEfSyM3dqWfivsY8KxpciyjNRAkyDlwXiAMczNb8fXhnBh/FXCr110CZkLFQzK21Hg+/oPDHkbjFg0LcoLlILdRMXdgeN63Z7j+MgzHNt7dASGprpRzCXRHVBnxNbEhF43hCxhw7i6g4uq5iMhJ0SVbJh4fQpwCJGGB+Xt/vYOAdfCN0VMnvk+g+8A9S+cwEP8f8F+Iy3gL4hzWzjkjIigugTd/xyk8134b4TcBRNujxwb+sR0QetAfMZsiWxvUQGM88WplDH/lp+sAz77oSMMXLwRtbdPmO+A18NcQgNMUQ4xTHH9RYfRIfSC6QTwOLTEtc3rP9yxH10sFqENWMFWicJ/30CohXJzHWZRR2FosMfcIbpWXAn+EXcZxgfj3pegbrAVXagRBJAisdlw5F9Jycr+ax8weJ9q1DX9IuaBXl0QpVguqsUyUWWlhpsW5DqR661qo1Wtk19HwJ3aNiOdTyPyX5wpDhVofTJmp3seqFkR1CTy57tiYMAGhhcAv+OVzdzn3CSh61zOkhXg6iypziwrhRYvg3UaUQNzfAXX7A/8FqmOPmENWyIgi2JHqsqrZDAjLY9PDpxuYjj3rr/gZT7dJCBjgRuPExAlpnSijm5iemNtSRKPsUiJ4u98PoaKG97iX7oTBEDhRtXAbDb9qyk+nMSIJneSa0g+MRITUkze2Y5XnOSkFfVY7IJZaoJkTXQXfN/h3YO/+4Hnmzb5tnN08m2HRKRhjyVLLv1nSpAJXeEzE8YZuwP3FsAYJpmMP0ZmePbOMmyi2ruYGHQ8ZizLZupPbpIg+UXM+/CTK9QinP9Y/wRpvOok9A7XOp5yXEvo8ER7koRd3/RdGO290K0Au2TifiDMZHrcZplwbYQBaeaTFOWfMDaWT6E2fRAJMcEbPGtqGzuWlY8NrDReTVrxbb76rysbmyK2kwzDdKllrYc3uKN+4ZgeLGwTd4GtOexH1g4+dkdW4OH/8WomMiSCJRYr9s2eRDTHBavclmSJt0jchrtSaJBFsyfaBF90dsKbfsqFyG+/sDVvtJlGrTuWRUcem9sHCYL1jdl2DB3T9viP6ZZEBee2JSK4b2vu6it4zGL6TMBLaPHkVw5wUAW6ffSuHdfnCZJk3AzRU0QdRxQi6m6YVF83mMUvYHAhkbMIQZeLqGtxpBULv/ACrR1V/cKlvwmJpJfr2AdOL+aywZ39APebgTE2JEwUhLdIi0jSxksJ6hqN3C5IkyjPZZspClfjUujpdJkN7Qpcrw1uyPPy3njv2nO1ZhpwpazKY8d9jUz2YDHqOGj2K/mVJL1SXoWR03GHwQ15I+H7z/TwTIn1aO9gsnfnYrqeCxCnGwb4wCvCbRCYDwI/wsXtM+1JkfEDatj6WCY+cojeFkWr5rBJh8mh5kYsiPjDMXOcbzO3eBI4TKgl0fJaHO1nF9RaRzJEYCj1VI0kxlLjD3G/fjAtoShdLvv6tK7yW9iM9i0bPe44jCHE1WyMxY+vJUJ3oCxD54LxsGyi0AwqnJ3HCVMCAPSUT3nH5abOsXcB0X5oS1yJYDllLgzs7KQF802o5bfzZ645sRblCd+6knYjhFs6TA/wgHH3IMAO5kYLwaSl5GZRu+rS34SdpLhVpi1pju87Q4kMuAlsS+UZC8rFxFz6pP0ETRmKfT3SUO+CGsA8g1r5yApjmAQQQ8dlXbFSejFOjBxXDr4JuPUDB/jhRkdEQKY9CiKLOzANAxGK4CdWvZ9/kQhnCxe6Fam0vaAE75eu3EXCqNDiEs5oirzz/eICPZNZhiiTkIy21NbvXkupt158DBx/02OuCT3/GzPCtVtc3vRaqhKS0JaCRWrh19POzsnu25DscPNzXEBR4jq5E4BCjWuTJ2+CNzECn5oL6YVUn6XSwm5G+xs3ROHEJRz1ZdfO+f4A7aHBd7voMjEowRJeQVJzEi3rYhjJaIOFYXA77r0h5rOhujb874qVTAonjLQjuGCoIGYkbaddTeo1j7BhByFISizT8xNJ+Af81NAKebOe288L74YG9pe2UpPr5c3cjInJJV3ePLM43TE3WzGOEbUXhDGLIFOyyyP2BhtZJhX2I7JNoYeK+WLMTvH6pVc8wwymatYbXZcEyVyHdExAwjwIT9Nu5vmuE1xn0jxV4UfliyUwjmCiuAbYzhA+tvH2C4fvhrSZ1Vby4Y5Cc9j5lE92y/6wHxtMQ8DXfGTQ8nI+MnP5Mu5+jK1kPraSaOe6WmB1ldonpYaWMs/NsvQ6sCwGuE7sEGcE5pNrDl8wqgM02Ix17gay0tyRKWC0i+YJbjjq4/jgDfFVRBYEmWis2JUEF7q80fnJ2eVhnhRDM9BNk+5u1OMistblMkvbVVIgSrNehl9HsG5pLou+HvMdJqaL+P/6ywW1+yagdHBp0SkC7O2Tg0BP8sYobtgDTg+XoW2yHnUJhhF26cBkLuCr/4SVbWyVzFjGWUsRSSrcPvChSYlMIey0Uj+Vm41GPCa6/6l7AeMPV4uNRkGV5coLOhxtuj0f161ub+i38cmP5+Udp2c+V9/ABfj8gV3Az6H/CX5G9EUaYNirLVgnrvJAWjsSTZpukvgVK7/mkkgTxqh2eoZzP2R6guviolhA80Pq9k27INbwVm30aTO5pGJOfGLeohUnvvOBwQJdr/UUVq/K1WqtZpThZ6VSqVV6vXK9ZlTKKrgBBmAB38SFQKmrigzdptQwnDg3xMjDPYHFnYaUfK8wCdEQpSWExZ60Ndznl6gktXhKcmW8hPwjGvr8MadAS6cLSzdOv0piOzvYubnGSpQWi4NlhqM0KRvKMV+yOI+RJtv+6eu9dMnwziK58P6kVDnhcK+JoElgiejJOJ4juc7lOBKfCKBgvmmif6dcM74abk4GNKSeYxsFDEvraD5+tv/v/yFOQKKJ/0uo7eHynxAictP5IoWAmM+DLW0bEBoJ7LgGsBRbJW2bpPoE6E1ObpbAp5gU+YXHEMYq0v2M4YyFehr7grQ4HtvJQHjYPfynGLqtMB7uu5Eofiw5BvlDp57HfGON1PqFvsuupgYROvpjVx+AoNL2GcbCeCJB8Ia5jPN5gj5U6QdaM5KceKR/skVXI0B892sSLJq44kTIUWhji0y3MblMX3+ZJ5CI/3n3kkhEiDG8MdPP8/R85kk9iWzPLH90+xXZCeOT6dJfgfW/n+i2ff2Fd6fQAr6ZcF6Poh7Mk8Qe8R1895Flf4iPkGC6L6Un6woZzePtAeq6F+c3gH2on8JVaRJ9OI8tPq8U3LPNB6KO2RmKec/r/6amR24s6wBTWI+sRDUkLj5PeB4PvJfou9S//tIH5sybpzBz1Z9zj7RfwdVp+/Tfh0fHZ/vfn3dOEyZyjvRhbsS/n75xkQiACR0wgsVu3YpIiqghVVWFhuwqKL8YtnjR6UytOXPaNQJah0P5+2mVeQFWmj+G9jGY2y5urWM5woRMLnIluDubdCYSDlFKlCfzJFbFk7jnAk5+nlw4fwFg9lDLmBjuovhVEL9QtrTraPDFp03CzXAxbOUN5nBCs1rkTnZ0Xh2pTlpmSTNLmmZJo51AMxaWhxVz40Q2zmmes+6yT2wI1gofKpe2O+Mvk+npcEcFs6xwJ0hbUsdbYRJVTGL5nADzgzHlyAp3YolAn66BKjoGK2jU5Uz9wWJ6BxPiu/BzEakPfnrXMjUQF6hPgxE6eYsKRI+0jAsd4DM0psevLirpieMZEtzEhajM9l6YY8FnnzkkGMyoDrtgtpTcVhV3jVJTa7VoyCfvio5Qpm9ONv2EXQRsmiSlsYwcM+j/4BPxwFy/p649U2qmoe9pn3qmnSqdZ/oB5wEqBTY9sLn+RMZU7C/bi5NP8ZYzkjqzHMOY3zv1r713FitP46vsnlzS7sUOZcpWl8lg2yxBapRrMucyWcXEvXh3YCIs+2TpH1muL5P9AbLHTP5A9TfkfpBiUepH3H/KzA9wfJDED51J+0Tbw25I+9xA9vBpH2jx8lkfUUdbjMwctHIgUdvGTX7ECMDNtwIfwz17JgtWOC2jOc6HOAnTZ7bL2hOXNMHq4LlaD1v3XJWNAH7wFuIXvIKtjMvcpQiParWVpgp3RvFlTKDAxVo5wmTxHdwFAAV2gNyiWOXz3fLznaaOG7pdn3NkMfEo0Np4v9H8GJgj5sFdZnsMHFVq+4AFYjp81KENpsH0njpdBXr1uNkqVm1qDVrp1WuVXpVWe1pTo2WDUaZU6opSrz1CtorPlTsnq5KlHzZXlW5dl01VzZPrvpmqdKkeJ1HF2zAvT7UqCZL59jSCh98GJgbUebSFEtwyFj7VkAy7wNcsXfLE6RJmKPeSvGOYWH2KuswTnKuON2Bj2asgOjfl0P7r/wjDT5JmP35SZgXyPbW5+R7cfX2/oBdUIJDIbRQB2cZBohpqAU4xDDyFKyyPC9M5+OwJ8lTgKvwpiapGlqh69PCqqlZA4/YPz06O9s53z/aPDsnrfVi+vz/vJAKtX1ekVsF27JTlRKT2ren5jumKY4sMpz9kXpHg6an0hambrr6JKy7MWnzykxWJmMERokrM47CQ6Xl8rhnjQhxIr3b094Y8WpqnE6bRHtS/eOpszl0WwqhZc1fDV4RjrdTkkZrIEqENnLy/fu+FZINkeafMMGaGMUuLZWmx9MyGXFmU2OAzYFFmg6vS3zyzkYzN3C2xkaxhZfIa4CHIS+Q1kOwR8xpY/eK8BqdYkNcI7z9hXgM5fl15DWzx7fMafGTm5DX4goeL37vrL24AjjIbrlJqgosXJRXeiTRAKOZURmIpSpGIKDeaaYmIqjqTiMDcwi5ze8skEY5Oz/49k0LowBRi05RnJ/uH4JSdzVC/czw2GjxxdgK143GzEzXFKFO52lN6qlFvqIZBNYOVZU3rMaUqU/bw2Qmh8XfNTkyUftDsxBwbuWR2Yq5c98xOzJHqUbITog2rnp1IsYqRe74zdABvHHnXXwBBvKUa/Rh8BJShGSaF6++c6/+xiuS9qQ84wakzGoDvA58O6SAYAmG/z2/8RPE0aLzxnoJNGZg3bcXLkhhp4ZVaHTEzt7IEDbAIdJy51Ah0nmTYXDZQU6tjKEVYYBJZ6LtXtyDuM65uhTMsaiLBgivgKuVTquqflZcAz+HPyEuo8l80L/H2+ovm4haZv0BWAtzbbYy3df66sTI0ObW3iVhZZ/dk/+z8pENOd/YPzzpknZt2b4MUyHiJi+BKuNCRF7ZBvcFmRPHC9T4GzuaObbgRJVl/e3R+svP9+X5MfHb0enYBWyqKn+oRhFH8P2Mdvkew/6+9Gt0vE6FOJyKiVQPzDgCtUirn1jTKO3Ab9+dlCSJLtfI5gsxMPZaZymLqf6+YelmtNRfE1FEDv/KQ+kRA4U4h9YkaViakrjZlubFETJ3TPWJQnde/OKouSBaE1SOCJ4yrc5ZfV2CdN/n2kXUxOPMeGaBEH7iYrbacvskXsZkDPv7CDw3gxvxk+6LYO28j/IZWTj80cKsiIlY/cqwr3RmZIqAPNA3xEMF0pP5pYuV8vB83WK5rak01DBk382usVlFZjzWrTbVW79WqDUV++GB5qMR3jZZPFn/QcPk827dkvHy+ZPcMmM+T61Ei5mErVj1kvsjaRZgB5gru2ZFgOQkAp+Op4jAfoBcvEJvhYeKgwSPX6THPS54nnkXEn2Bb/2PHTiNLfv2FNLLd3beJot64u3tFgX4FNbH6w0KgP4h8OmE2NmKkfnh0/kNn55ycdU7Pdt53Ds9IgZxe2ei64pHlZB3E1gN7445Rx3SrGm0efnhb9tRbiH/2ikbxl7Q7U/Nwct/viu7SzRQ8i1f97eNVSqNSWRSwQkX7ygNWkx79nSJWk1WsTMgKplBzudMtmo97ukXzxtMtmjecbtF88tMtml9XsApbfKfTLZrzQlX8SYjr/42Y7931Fy/wCrvc//nrh6l4y3jwqAOtG28NHbdxKkh1mwLhuRZqOfVcC/krONei+cjnWshUqfR0QzEqjCpNo95QqFGWK3pFrhlN7RF2jopZctdY2ETphz7XIs2uLn+uRbpc9z/XIk2qxzrXorn6YbB5lnTyqVN82Cx84FQ8gMrfqRc+dOoRbwwDshhYdrTFKh9tIc/d25cdbZF+tIVazoKfDxn8XKndWdkT3Cv7BPeyR0rMOhlhVPihl/a/z6kScuqpEmjoslMlMpuU2aSbbFIWyf+bRfLl6sx5DdlpDtOnOTTvF8ifqGFl4vi18jKHVAPVQ0TxgZoO2xJ9Xt5RFPjxvP7ag18qflRV04Afz8uvTfgKDOET3lRq4m70lumu78ByZVmCllPIggJfGSu4ju81wlu/gkKBKyBucP6VMf+IBZLwd9jOVFMJWRSfq29iU/28/Ea8xHzypTPRi8yfqwr8w0Yp/J861aAxNxGDGFOUa4u4zb4yfS6n+jQnJvB+glVzEasJR+smbmplmpuw/IFt4luGk71ZWcR0stRS7axOc+aB60Qrq4sYcuKlWihP89FjzJ9gtrB1ka94I6/yTJsCLZVdfWHbRKGlerExzTFyQxLMGouYJd2W28+CoePrVmLQagt5JV9ADrzwvdFw9QO7unRcYFYLYFS7gFrcq1gIYBBJUcNX3MMN58MiLeLO5LJaFHuet9fWESig80jGZmZQo1dpJfipi/hF9EtznDE6Oh5rOPKXVdmQfKmubKaY0gE/7WxZExcXuInf8/oe1laPFxSduv54wZIjkZSYABYqAMp2GlElJgr9O8e2rlLo1JjOdkJSQaWls7Qd8MvsYLhYLv769V4v0Y/iXmV8JdRRntZKdGUoT7nBaqymaYZuMNo0KKtTvdyr1xWVVeV6vTyz/kZZr7Fc4+yX6FuJpOX4u/RX+gkT/eAPhHn+V/7A9Ip95u/4vmtqsHSuS9yvkDY2NknqHgAsvGALgLg9uwNAZIAeYQcAMFx+A8DfIP8P7b19+h9HZU72f4cfeQog1o8AKzinjnjs8u/xhguecX+u1uN8ux8/e8IT8aK9C953cZ8Kol0C9dRdAtV5uwTeONBk+cK8wFjbnU6kumv6P5yoN6X/QaMeJ/s/FkHAuDCsk7/55ZQ4MyLGk2y2ASkPqWXaHzDoM8ZkfzoufPJtZrXltpnVnhagIscMoWYINUOoGULNEGqGUDOEulIIlTWbCq0CHK3im8YadaUqq2VN1VWjqStl1nh4hIoOwd0hKi+9eJt67WlBKnL8qlAqNvhOu9Rry+PU0VeGU0f3xamLKghxallOPRy58Zfdzb4knEXNWzU8y+fQagLa+UjyieWoq8scn49kT4xo66o85fpmiDZDtBmizRBthmgzRJsh2j8X0Sq6rIEPqjaNWo/qGiAcVVZYo1rrNcpKuVF7BEQLDsE9EC2WXoRoxf2nRLTA8etCtNDgOyBaHJh5RwRK1//pg3vCCEAmD+AtvWC26S4JZN/jwdWFPWdo2vhAAdl92zk8XyU4exAi0UaTNxLTmdDMEKKKpk4fEXibEiFgrVVTE6vNeYC143k0gS5vk1HlPTyDQKfGYbrQ3vn350e7M6XEwyMmtW+Jc6fJjzsnJ+e7R4cpDKhrsZk2nOzvHnXOT2bIj+H+NO3rzunpfudkVphvwxcsUde4N+6G+bFyuBtn+ori7rl494lfW1etK8u8tg7InhZ3I8cMd2e4O8PdGe7OcHeGuzPcvVK4u9Jr9ORyWaszpjTrVblW1puMNqpVpUJprfYIuBsdgrvjbl560Wtvxf0nxN3I8avC3djgO7z1FgdmLu5m3uj6iyue0Q3IiXNFoUnL4269IB4AB10gBzu7O4erBruhfRFEs3Weiw3gR9jOFNC9JL2A3KqcDrlr8yC32B7tRXnnC3MyMfyeJngsgN+8p9PgtxgO3oC6c+vM8TT5+6ODg53Zt/PuOTb1YbWYAclH+6fvd0725sQFUl8BvHd+fHSY8gJgqn8UB7HcC1Kj6q8apOaTeDUh9Xwo+8Ry1JuVZSA1kD0xpAaOGaTOIHUGqTNInUHqDFJnkHqlIDWrNTVZVuvVplqt1SpGrUbrerOsaNVypVntJep5MEgNDsE9IDWWXgSpxf2nhNTA8euC1NDgO0BqHJg5kPr6P36AZ2fhSUSW5LDggh+m9fd42Z1ISfMG4uvq0GXjGWnRzAjWzr7x7g7lBMKuNFKfFq7P3YV96gSuzrloJniJiCInIPY7sLfyt1SjlkEfYLP1DPn5/sFB5/EALSjeygFanEIrCmjnAsknfui5Wl7mMKxq+YkPw6qWMzSbodkMzWZoNkOzGZrN0OxKoVmVVqpaRZVrtKo2qyrVGz3KerKmN3uGQWX68GgW/IF7HIYFhRcdhsVvP+VhWNXyV4Vkob13OAwLRmX+luzotHaEr7gr9q+PXw/iR4LDpkXQc2LT761oF59qNTcTvPqnWlXLq4Y0UcVXE2jOBXhP/ixyY7lTrRpPfqpVI4OaGdTMoGYGNTOomUHNDGquFtRUe41Gs1duaFSh9VpD1mpUN3qKqihGtaY9wrnL6BDc61Srxg2nWjWe/FSrxlcFN7HBdzrVqpEBzgcBnGUl9XiqyldwPFVj1YApnwyriUznQ8KnlaPSWOZ9QED1tMAU9xdkuDTDpRkuzXBphkszXJrh0lXCpfVeU2P1Bi2Xmdwz1BqVy9W6WtEAphrNBp1df++NS8EfuDssxcILUKm4/YSgFBh+VZgU2nt7SIqjMgeRirdbAxjFJB3xKIHFwCcGtT1+AHMCrOKjl0uC1d23Ryd7O2fkXZEcFFcJqr4TuBBazCEhthn36iJkxHbjRWpHR1Cl4lTshSlM+9CVhuBXbqbuCpbngd8D042PfuYvIb7jqVd86KAOvov4oPjE0BdUddWQL0651QS+cwHnk+Pv6nIp2eqTp2SrGfTNoG8GfTPom0HfDPpm0HeloK/eUNVms1zTevCjaijNnqEpVaPOGnqF1g354aEvOgT3SslWb0jJVp88JVv9quAvNvhOKdnqDQAYSF1GLsxlz4Va4XzsBB7l7Zo6hmlZshCIVhtf7UuCqquGRbn+ryYYnY8Cn1gOVa40l4GjSPfEeBRZZoA0A6QZIM0AaQZIM0CaAdKVAqRavalVNUy/ltUywNNetdo0aLOhqRW5zGr1RwCk6BHcA5Hy4osgaUjwlJgUWX5doBRbfAdUysdmDizl58mGeVhGDhFC0YCcMagXm/bXR6m8gYncKH4Omwmf4oZOgdbbljI9zW4369CSgqrKBVktV+sFZRbiVtIgbkOZB3F/cEwdn231eZKXw9yGO4lt95inWwIAx3ne164TXDL3AbDtTNp256fOwUHndKbAWxgA895IGDV15aAwn3UrioXnY9CtkmY5+oePgeOz7VzUyZJNLzTqwkANcCy2dFBi5sL9GyTe+qZQyK1FGBLsned3EV+s44+N3NrvOWBp9gj/XkRtL8I0DhjZJmW4DfdBKmox11/PHwbD6y+ug3s9kBDWJGdErr9Y7OL6C/kmv8HbM66nB1bWWxdXJ1YXuPB5HtstoizFVqPerVnG13AxhUufc4UC9FDU/7yrI+3lvQ3L4BZWLhYxCT9KhPKebEvTCjiGp/xAMFV+YWsuoO++DfagrL6w0P8AL6VNJTJkQAuDOnI8X4L1H9DK0IT1K5JvcpRwGKMlUIqXQD59C32XXeECKIkFkF9Mpx+57GKaHK+ZDqzo2znetVumPQqSyiRFqwD4VswFvw6opLA7xGc+bDhRYer+xj9sl8q5aJou0UtYTVud31nbKW2xQbREM2wuaeg9SHK8WPE2eYE4XwBtHEz120pWvp1kaGwTkuHXNMngq21e/+EKlY5EK+Fw429uY7dK4SSPv0emV3xaEwZj4q4wINxQwsQnPVjZ9/Z/IEMKLh8u+qBVYFZcAoofm29xCXQ9J9b98bWu63noCuai9WCGAC0oAPQRdxjn2mZ0CPhCylcf/gnqqOe330MDYTKBC3P9pQ/T3cN+INDj3mgzZjqH6yXTPNNnyBkYh/M2JODHHjpd1NN8KNPA90etUknc8QcMIHJxaHoesDdYj7/ooNhzY5/0vbiFRucNvwnCTl9KEXZtnrSjoZYfK18ozOXlZbHvOH0LWRMfFkfmt7uaRe0P22+oCdoR2IwgvmOwgDBCL5hOvuUFkrzXEh3AESEwCwd6qvXI0DP7WtFmftzUYrGIbDzHCvgaYZka6qWDZ3dSArbXMKk/uP4De6xYzEdySkJQaRu4cWliNRExXaGGJDepkaVQ61AjUQv57UgTATO6Tn686IkLYjRFsHVrUE4bB7iaizySsCNG+JaQ8YxJ3lHhTmBxBuALBntg2MBJwnjusxwhW5aZ6mXkt3d0PWCmhY3dKgHVDPU/gCbwXeaRSM14r3GtRmIgB74RAAC7C/of5KNboqawd/Oid5Pj502rL5oHYMYu4EdRd4b57V1+hZxe/4HXJgRdkgHMU71o6iNQSP65VDRgRPZt8CkBrpBd0ASHnygqHk8Ex/QuXLSgGOjW1VJMDq4ce5pHKbBE708Nw3yuofKnzPj0eZ0+wIB4PZwjSa34VlxKFOLiTVvr9GnAFRzngXD9Ir3XKNRPg64a6XBKUUDAgJptYcxFEuUGZ9C0Tb+LqBQ8pLVSKbYr7jffxE4QER6oY1wJiQf+0MJPz3LPcv8fbC3VecVCAQA="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514859.56/warc/CC-MAIN-20240718191743-20240718221743-00004.warc.gz", "offset": 110658803, "length": 7228, "data": "H4sIAAAAAAAAAO1d6XbbOLL+bZ2Td0A0E9O+sRZql2xqjrck7nFsT+zc3k4fHpCEJMQUyebipfvkAeZJbuc5/DT3LW4VQFKkRDlK7HT33I4TyyKWQqFQKFR9gKBvd9/s19Rq/UnpW3hXubj12ID4LPBcJ2Bx4gENIbFRb7Qq9W5F7V001EGzPmj3fogLvGGm61uVo4MB2Yl8ZxBF3Br0emp/1KiPKuaoRSst2u5V+qzVrzRbzOh06g3MHD4p7btOyJywcsyccTgZkGa93e7PkiVD1PNsbtKQu05tEobeNpkG4xCytDlWv6W+yZ2RO8cLNWidqWaj0m511EqrX+9VjLZqVRqjvqr2O6M6/BvGJKBlM/J90bibJdKyaN0c9RsVg/b6lRbrtSq0bpgV1uqaRoO2ILOXEDk6q+xaFjAXDIha71cbzVZV7XSqjVTQ1B+zsPL2zdGAYI+CQa1mcMPmbjhhP0esOuVBAN212MinjsmqI7/G4emm6k28f9hXtiaeuBPqAWPr3NKaKe0z3w1d07WBcmMhKbQDGO9m0lnuTZhfOY84DvHF8bm+/2oX/jfq+tnp8fcqjIZ+Do/tTkKI3toutSoHfMyCcECCCQVl2O1etL7rNLvnrZPDl43ji7evDhuvTi72v+/+8+LoRaIme7ZrXs7X/Gf75cle5+Wri28vdn84PHlxdPTDcXv37avvv2t9s7+XiNOC4eAjzqyUgUXFuJmEU/v5zdR+UnpSenVxcQb9VEFt6+RJyRIqfDGJtojaI99EtlBnkigyefn64knJjHUuFKRDdhPWkOQ2MSfUD1ioReGo0ntSuqL+7YDsmibzwsqhY7oWd8ZQnZoTVkEiPgrac4OwAinmpVbfIp7PMg/Lsp6U2I3HQW0G5JyGW6Sukm+og32ok3p9IP5LXoEdaMq95MDp2cR7fbuHylM5PdvdP9jTgBXWwbHLqtQ2iYlrFxFD2hWQQwXk0Cb1xqCdyGGbvKY3ld0x05p1+TPX2quz88Pz86MDzeh0qdrvt/pqy2Ssi3MJZpXR6Bm90WhkGdvEo+FEq83VnxqnHjUrx6cvj040i9ksZFaGNxwj5I06FbXfTfut5nmbZyomipyd7L4+1OKE5aWA/267U283e91Ou7jYwe7F7t7u+eFDmLQpDPUUNASV96MaGDD/ivkD4jaCax6ak8qZe838PRf15Qrm7HeVfZ9e2zBjE11lsfoNiOEvGtNGv9vt4HQo7Tw9ON2/+P7skKBKk7O3e8dH+6RcqdW+be7XagcXB+S7VxevjwnMmVrt8KRcIvBTRtMElun6+rp63ay6/rh28UZONFXFSsn7qhVa5WFpRxCHKegEWkFdUJa+rFHGQgObOmOtPPLL6QxTxBRTkBKj1rC0tjNlIRUWsgI6zK+0cnaWQkX5pJWXTdcyqaVkHDplWplG4cT1M1XPXu+Rl74beaJsvvAlu72GtS3IFI8tM7EYkbZ5i2Sn2WKDFgtMn3topDJk9uI662zs0yu2DVXJ6xnpF4J0AUe+a7hhlh9q23GbTyuVo0PCQhIADeaMmI+UTHcK85Ab3Obh/1YqRUJVvqu83YWVTxa0mZJSV44OtUNrzJS5Pl1xdu25fpjh45pbMNstdsVNVhEPW4Q7POTUrgQmtZmmbpEpveHTaJokgM6sra0B6ZCHNhvu05Da7hgkwRxi87FTIJKdmixbWsN/OzZ3LsFhsTUlCG9tFkwYCxUifANF6IQZBAqZ+GykKdWaLFMDkUxhxZB/qlDiH2q3Ue/0G61OXZGyfAjdEYikQq9Z4E7Z41N3PeZUAuoEj0/aF46cbvHAs+ntI9L3pkZLn1ojaMh2HYflee/2HoX2GGYC8znTvQl4O1+kieTNF6Htgzf3JQiHdPxFhHHFcWY6LApYMXmwRwQydLS46MBJR42ADco2ff7q9M3F/tsLcrR/epK0xqcU3MRaAH5pbUTBpMA8hRdhhqQ1lWxKu/+OXlGZWiaBb2pl7ph2BHY3k1OzfDrWHd3yXa/6LigPd2oy4wEkJ9SxbPaoJD3Xi+4npswRQ5kQssZHZOOp5ZrRFERdheDi0Gb4Nti73QcvJDgBu735K6nVwP2MfMIZWt6PlCcawVKjyDFx5DYcd6qbmLn5K6avgSdMYISmgeawa7Lr+/R2Y3M7zTLdCBaGukwYuT7ZEBUghfCd4qYv6Bgb3lD+S9ms2sKNgcLPn2+SX4ngZU10dIXKyNcG36yas85oZNYBInsA5JD/HwWrz5//BD1enfa25Ag9pffynfzjsxCiRikZ0XmRjr8g/ReuFCahJAphTQ44LNNiSGxFeHOg+LDaBeTtm+MA18HZSJdmQ0FEUaZDoQ0IbqgcEBQvsL8PDtA5uAGZvqBPRM2QifR/LEkf5NPBhxLc89FGlqhGym8vXoBnFStB3FvJEAS06Ea4DhCRjAkS75kdsHxx8AA8li0iRTTT+tInmqfCOe++c2sWf8fDGpigKWSGkQOGIP5TFQTAVn10gknJvnPBwx7xMUgWO+OhjE6dY4hHIV70IaqCRAhywbUZEGXkV0a+gkk8OGBGBD76iIIYMAVM5plNufMNeGqzqhB0XKLZG5AfhazQ1QJCYHD3aMAEKUEeFWCgVKs18V9Bwf2EmRbzsK4CUbEw07XXLAiQ4Hno+kwRZYA524B2BumkzhWSY3rNHcu9rgKJA+gz9va9GKM4oTqNq0AOTvwchY1fI98G7mr0HZVoBYQ/ETh7+LwOzLOxltRXtogFAZUZHoGfOBDSeS+0AbsE3EKrM324f4yERb1XC5IXMK4YYCR0H4MsyCWlvCrDQ1QpF4ZcDybutR64JjjKusNCiDcuCdjIle2/5LGIw4BPwcLoAaO+OZnjbiWixR3HyEH3MdgJwnmqIFgZupV2DNe6Ja6DcA1GB0KlLDaikR2ehzSMAk3B0EtAKXe/BcSLICIyle0yIRxqgFBRPOUVZmdqFUfQyokLngaLLfcGt4RKo7AjW3Mi294WdtikgbCyjigdELEqcOxSKbGjU0mpeEnYuz2yNhRmK8+59VzZn3DbUjYlaUky8aJLwoA+TcltrkZZEpEkhQFOqqeGPrIDQSDOWLJcRbakkVt/CS7AUD9eYYlcYqV9hsYg50f+ExLcDUOfG1EIhDj0D42/4jpjm4WBzgPD0uMxex6LOWUNOJNU5OK/ZviMXsr371NrT9JVASvZ/D55yFZFo6K1ZGV6avNNsnL1DLsJgaS+4NnmVTDqUOsE1rLteNGOV6wINed9RtnEvL2e0HDj58jlEOxa0quQ2I1IlEIBXWNEOfzu7Fg/Pt1XyEDIYSmvFr/S2Y1n62DrFfIc6VbFAliNVQoYVQwEU5Xt1SgtpeLAQq1sC3t7D5lYeFlKGa+KKBlN0LkD7gS/+jSq2Z4upZzQLeX1aSbcxxGsFMkD5brS6HyKWHOdfzSRzgZrTqixmY4FipqsEXythu4xQpL7IPSspy8mT6Hx3S4lPntkp053GjcQYSeWmC6bx6arwHiJ2kXmS8xp6dN/zITpynPsU8Z2JdEA1FxtJD6mJLMW7teRnIc814cZJ09nrGNMr5D1dXJ/MYicVykWROPxCsUMGlyCBz6T1griytiDtNYykd0zbFWfgdxMlui4soXyVTbvn8DJajP7+34tNkvv8xr/Hk37ozhHIYV+ByB6dH8/xSFcgeij0ZvywFzwB0Gg6HgpNMTKKFZN+aVyhPuMA7Ver2NpKJMpKsB4CGOYXx6maVNImD0q+KijS8h8IDBpDhPg++43hLwtRmxahPJCSdlcjrA+cf1AH7luiG3GY/sAQSA0pSeYS0YcQDNpluOGnh6HLHHHhLqDi9ooJ0zmWFUcmHhjCuEQRhoB1HfGw10CrkXIJSbPwpDBM4T6V24UYPx/xX4hA2hfFi7t4JAzIkF1BcT/DLgLffi14tZlI8IeuQ7IhPugdcA+Y45ChjtUBsblatG+cTlPA3epYkcYWgk86gzfsNAFr4f5hEa4RTlFmOLugwmjQ+gVMwnE49AT3+F3v/kzP7parUIfkMBOjcJvaGGohXwLHWaJoBAaHDF/iq6VUIK/pSJDfDCVvgK0wVboQBEYUBK22dQLb5Ussb8fQQw+pgb1eVjFfaCtK6JWm9VGtUkadbWDRxfqXVLvDtq9QbtL3nnQOnUcRg5vPPJ30SgOFWh9FrMzgwDUrApqkvjzuhwYsIFxAsTvmLJdel/KF/Td68ViFUhdLGoy2y4oi8lgnTxq4R5fxefjSTggbe8GKexIQBbZTlRVkGQwI+1ATA6cbnI4D+4+YLKYbopQAYEipGMFBTPTOkNHz0xxpJgtEjCWKFL6LOZkrLxxlnjQcwWghJ+QgRnNw9u5dkQRK5ng2Vbj4rnRyHGRzxmmq0524iaHJ2x2xexGptCaFBo8q909mQrx6ZuLKjl7dXpxek4q5HUk4kTRTym3T//J8jKvMjFvY5QIHjGAoVzLtDF7m1jjxZxVmkmo67g/6AbMWrWZ4p9Svlz2QVqB+J0vFSS2BthMrmi6BmU0EFc+sQG5ltHoXLeyBfWQhz6M+2l05QuMSq4wRm7VYQ6ZTQICQ52dCDnbY4cUyJmThItc4+BoU8fate1yam+V2eIySLOFM7Chbm5LqCeLyujUtjfiDOG3X7k8gHUulwvN8uk4MX7wVvfsKMDf2eImN0xkk1itOuYjhRiuD0ZaU+qK6JHMhlwlts+y17k+wYPJ3oiunwsmysN1xwi87aLSpmvb1AvYUhlkCmxsLvZj6nInEC/zPUkqLu1LUuChvflc1yFgNjMX8C+pxvlHEe+gCuhjdLZdPxT7JVkYDYOppHQKMCSl9XiPfcNitkhArCGzhRHHYD4GYfcDr1h5PYgMLSG97tNfJEfK8w2UgTtKW9kUvn+EZ9DALlsK7hvEWWRAFGXzuQK0PM/XgZtMfcHbQlVITWuh46MzB/oV+YEGXsmz5kH2PNuzRodbkNZs1Gee/BrJy7CaiA46vlXfUpQtdSuGUmcig4z6ZiYYWJDxQo2N5EhhVr5Lg7yRDzGPHiN+4CbhuQjcIALHwsfzNFqeZXyDOrYx40mMHYZz98FXS5opjptj3U2aIPJHBNHpBhzfEbvC8Q5bNnxeS+F/dkXtDSyGIRlyPUeRpMHV7E0q10tuS1XRBe8b89oqsrAb2id2eiY4BBVSMpsz9tO0DJYHyjJ1r5jAaTOVFsLExZmcMSYQkZ6LSe/6wti5zj6EuJeaIpQIFlbmw8AuTlsw4ITaoVa+8DmsRxz1C4/ZBGUizrIUZcTxlwkTBFxiPE4Ic4P5yVKQt5XCMBq3Ov1FWkqKZ2c0xXDD0J0qZCKMoKY0F2yoYBM31/MWFDRlKg/6KFNTBzWAeQZUxchKc5iNKKauz3S5VgZp4Jh4siIaJ+DnT1xoD08+YkjEHS9KbO6EWxaGLLI9ue79+JNCRLOQoLeU2vCeGkIuel3Hgkml+2u43lzxw3/dX2HEmW3JOhnO6KAx+DUYqN3B+s+RG24HzOcg+V+YFa/eMnk7GDTUuAgdqFilEz+eH+6+2X8VFzvZfp9WUNWUpnADkKkZtXo+E/wJDxxswWQQl3qv1O4VM1rgtCOqKFzDUV919VzuEdARmnxfR6eJQQ2W8QuympPpmY64kqWBhWGQnUpvihvcQE6DX12uZUo8YZRd2Qp6dmlDyrAoNavXAnJDAWHUlFmol+8s4Q+4qrEVChZ9tx/vzY0N7E9ap9vobpcWLEwp6/SWmY3FzoTRSkMaSbsiTVkSPWUFnjRuMc/mVFqPxDLFHipuH+NmFZJXtsR+M9ipRWd0QxEllrqjaT6J90TElu12WZxAwSWmyE2VTlS5WgO7CNZJDL7jTuGthtnrrjgZ6TBbU8vx6UI+PbwpZ0RyNB2nptKSkWw5MWXlejkxcOUmHoRM7WM5tY9o4XSqq50btYMWsizMsXIQrzzysDcjMI98Pl1n1IxCtp3q2keK1ZaOSQVhL1omePJojCODGfJRQgyymOypPJ4ECTr2uJyfVAHulyJEAyLK+7kx8KLJkSGJ8CXgposeKMN9F+YOrCR4NhTcZonYWBH0Z+oFBZM4qS7DbXBwSY28jG7J3t2/IRYuqvBDs9/rpRIzwxv9CsYGUqu9XqVRr7fW6dTb9kchrib6aBpq+DGNZ81dd8SfNV5AAry/ZFfwOg1v4NVw3cukDngGjs+0XJIhm0p69qxRZyG8QO+e7Tef7fagh/BoRfAieplW/Ox6aDIDrdHtQ5aXJuMnCzS13+2sY+dgINMcD9zsczYGKzQjIjBsDWY4n6XRCOE3bQ/b7/VBvpkcAeppIHolHfXcn3TeGT5J9HAWTs8ZFuH8zNsVUybKVbs8pf6YOxXpOAw63s12dh3HnfmcvcClg4TuJQOvoGvUe1a7x3r9dt+wDErbvZFKqdluN61+v9FQCIoq5Lj6qN2G2mz2Gu0egppLgU4BOEW28FQKdp2lxscKWLQrLQ/GreFhwwyNotoFGzyzVetviV6Xz0QJNK+mNK+zLWBFHqkH67rMRsrKcjmyeTxCec6QjaV8pTspRZwdne8dFPOFOfdwhdl5nkrSwV+TOE1kS8BmhiQpvns92wrIYDa44ZWV7ZwnKBbf7WTAcYwTgmFqqBDdjp1XAXamgjDGlbHPbvP4iwLq9LNuTmC+KMMLhH0Egi7nB6gPqlCufCzJVayidHAG5Bj9y7sPY5/a8lie5979NmWy0oQ6AYSKuTYF2J3v260HMc7DOgfLBQJZSayUcDff2+wCdfdhGUNg1rHfD+JoV9IgizwUensYJ8jVJPm8WqvV6itkmF1W6HCLSLrLOGeW+iCuDy2O5AvUZBnTQmUCMOIJ32obwqihMOHQ+djAp5sdBTzfgql8mKgd5+6DGH6pvwK0XKYBuBQt48QRS9iDeDma4qc+cIeuYORhbSTessYDCIwe1PQLCE9oWNRsm7SJOV3WMH7KKHpgt48ljUW9wd3Ku/+hoAsfretCo2DD7QwZklZfxryA7R7E+j5F8wWNs2CZzixVftF6OmHRwRvuv9o9EVNVuhyrVW20WjhnTg/Pjw4zdZf0OUECH6aoghsC0YIJQYDNPrnvBZ+vhT6IrQI6lKM9LNwYWlwN4s2OUrJmCo8tu2rO1v0C9GT7EVcq8mN+hfgJHPvHsNeSh6r8U4nBgc8xqUlnMnZ1iwj3uqAFYW/k220SW4FqrFsfmY3xICXNfdY0Xkaj0MH5OpH//0zk5MjHwgQXcFFptkeJ81tsR+rsBqaqTfHTw+CFzB5yG/DJtjmz7XjLX1MaszMPGRL5cKkk46XJrKRnx0duJIhjGmhDLFYxqC8aDSf3l3dD4R+H7L6iIXijus0NYBdKn0ceugb3VUg+uzCrdIwfluBBfBZhec2AmfmK5zIhqTM8iLFz/JCrcHwnCwrJrpijZM/PpKLp9Bpqokf5TCEHoSr53HzXj/mVz+aLFHSWkTMG8o9uSAA+6GvqOwu1Fjr6mo5pwJ1C9gIeRqINUCkw6JEj9CcxgPIg0UG6qZCeLSIzzS1lVTddkgr2oXNR6WJ+YTyYR0NzFHJZ6TmeDHLxe+GyaquvNlYAZkW5L4jMCvr3Q7OyyD3YbFLgdwRnRZN/LXRWdPnT4Vk5OEvw2ZfM9fGoyp4PmcxZFZLdtb0J7lSTvdOTk90Xd//+MyGzcZ+eNepJr+Yg2PsKSKxV7ddXxFpx6w0qnLkCxO0HnKXAbuz5fjYsK0QrsdkFYDYZgIeis0I5viw8O2KqUe90rGZ7ZDWo2e8iVNvrjNSOOep0O9bjw7Oxxn8uPpuv/qgA7TJDuSJCu5yzB0K0y/j6Ihht3Is/O0i7zDR+xVo/AWvtt5sYYResF18h1y8LueIhxYdF/UBAxgSfMvzYbBqzq20Ebs7cuw+wLub3LWg2GPh9wWJY3P8IsFjtfIWKvzjC1GghwrR3dPryze7Zq78kVJTrnQf2yhXnsx7UvxN3Sjzf9fylPUtWSRIvm58KQhc7BTEKPb8UPwBMXmVB+mMwZWGWEkxZGIt8/sYj2NxN8hWl/mpD/nNtyFf8+/8X/q22673+VwD8XgA8H/R/FgKeJ/HngcCbare3CgSO5b4kBI70PwKBiyL3QeBxgd8TAscm/2IQOHb5MyBwMThLIPAzWKSA9QkPQpf7yb0AU7k04uc1DyNYtdiKyPiJ6xvMD8nB2xenb9/s/+vPBIzLnj5r1JO+4nljfLEpvMQ9RvjagRfZ6znk/EEUYmi90yyC1luNwmPMx9R3I3AKVgHME5EvgOXxmDwYK0ct+rJYudGrt1v1ZrffMc1um43qqkUNyzJos1mnhtr5Ali5nBqfjZXnqj8uVr7Eoq6KlS/l7KFY+RK+vgxWLnvxZ8fKP9GGfoXQPwFC77WabYjjFheW/wgEvd9uZxD0xJz/mSD0VuMPA6I7zT/k1HL9KxD9xUGkbgv1/vXb86N/vf0rIkirgb6Fq1sM+q64pjwAC17Bsj4MCs4Yv/SsdMYCIhYMfm8RFoyWIT1fXP96uvjrlPt6RvivipE2m83WV4z0fow0F+x9HkaaI3EPRrpTy17mkL140RCO60T0z2R4OcjHbxDHKwQzNzuHLAh1vA5jA182S2u/4nUReN8MPlfRIa2KuzPIkKh484y42wYMjx9ulE8iWP98V3i+eMEjyNMjdx9sdnX3gTwtp7dxx3RGECMF8X1AuVi2FN9XVNTszorNGjT45CbTNARixZXX4kLG9L78zJ1yUtp4Q3vmxhh8u/yymAJLC9Ya5j1+21OgNdfxlk9t3Y7DUI3OXR5DXCeIjCkP8Xo2yWZ+sMT1Mgs35AgISiwjyQ05IV7r4sfX0SyUx+uG5ouLK4jAbYAaQsJqTS2q6oB2zVfFtOKGEDObL41pSnojS+K87dRibZ7fKpDv1iRmnxZO0rIXbnKHHBz9N5m/SBUvv0wRtvRu1VK6PsVpui8uw8Qr/iVkt1AAv7BDDyJPXhW5/AbSkStiOgERinfoN5SHr/FLgvBTaja4ItIHgtVXLuZpo0tavWYGflkQtgwNx/oZF5B3/+monuWYp/jL6T7ylZsppr/4xWzDgitrF5hdW8atNzVw1sQSynxT3th1xzY2TULxDaGabkAseDl8gWs6iRw2u9hGXsP6UlTItr2WEYDNmYONJTfm5nuPDQZ8bFQdFqZdrVar2Ezg2pGwhbDIQMvyC2rA8b/7YHEaX6MIRcsJn4pkVBlCa4KbVE0EAhcrJCnlNbIWax1qpLyCNYP1iot3y8P5m3jFmMX3jIJHVHx18BzGV/bU8vzik+TgYX6JzJanzIkOGIQG0JWwPHxSImTH5oV36ZaHu6YZMW6nYONC6b+VEXbCj2rmLp8UWv1EfL8itpvsn4BhAf2PykmWpBRLtyylmx2/hW+MResAjTFwM4Kq6U7Lw32RQs7vfrvyWZBjdMUG8As4qtz08Otn8X2tasGIHDkBKEsUEggaJq7N43ucRYT1Oa0YUTUy7duVGjm+RY8u3waCukL6c8OwvNVY+QtmfPG8Lh7gGn6TLX71bEYrXsqkTCXB3ry1Lp4GQsHF96OtZSeCQYE+jfT526azVdPrc8U0kvXvd3rwuxl13NQDT2CtVkvtiv/0abrYE8EnfpGL5BgvVsZ3+J2i/weMGLU7O3oAAA=="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763861452.88/warc/CC-MAIN-20240725175545-20240725205545-00033.warc.gz", "offset": 3636501, "length": 7712, "data": "H4sIAAAAAAAAAO1d+XbbttL/2zon74Do3oT2F2uh9sXSPV6UxL3eru2kbXp6eCASkhBTpMrFS3vyMvefr3mOPM33Ft8MQFKkRCmKlzZt7DYyiWUwGAwGMz9Axvfbp7sFNV98kvkennLnNxPWIg5zJ7blsiBxj3qQWCqWKrliPVeqnqvNVrHSKtXeBQVOmW47Rm5/r0W2fMdq+T43WqW6XlSrDZajzX4lVxmUmrkGZTRXbRT7+qBe69fqg+6TzK5teczycgfMGnqjFinXG83mNFkyRCcTk+vU47ZVGHnepE3G7tCDrM4Mq99TR+fWwJ7lpVJmNUNXc7WaXs9VGsVGrqlX9VyjWKvWS2VWNCr1bkACWtZ9xxGN23EiDb1KK4OikdMbJSBSrUP9Un0AAqlWjGKpadQHekhk/yS3bRjAnNsiarGZL5UrebVWy5ciQVNnyLzcm9P9FsEetQqFPu+b3PZG7Bef5cfcdaG3Bhs41NJZfuAUOLxd5yejyb/MS7NDfW9kO5rL2HNudMowOgHlE8f2bN02JV0YWzXMoDemTY3cHh8y12sRd0TV1t7LtwdvD093e6/eviy9OXn5em/3sFz98cd37456hyeVWlB3x7T1i2TNndc75+/OT87fbPd+3K7svn231zv7z+vv3x4cnf2w8125F0rCAEnyAWdGxMD8mF6PvLH54npsPsk8ybw+Pz9BtkHjiuT4308yUv/OR/4mKVXJd74pdJEEWlglrw7PZxXGY9deAYm2iT6ijsu8ju8Nco0nmR9yuw69MpmTOwfJugN46Fm6bXBr2IKyvnXBDEHNYjry1iIXjE1y1OSXoGRvqXPTItu6ziZeVA+KU33EUHM8BwU/sV0vByn6Rae4SSYOi70synqS6V1POGhMi5xRb5MUVfIdtVAGRVIstsT/sqdnoDe7tn3BoZ8no8nhzQ4qTu74ZHt3b6ejU53VysWqFlenNmGSeOfcZ0C7kQMp5kCKVaRdCqXYJof0Orc9ZJ1yUf7MtPb65Kx3dra/1zFqxSKr1UH9Kw21WqGlQaPUrKjGoFkzaiW13yYT6o06hZn64/7xhOq5g+NX+0cdg5nMY0aMNxzhopqDfufUZj3qt5rkbZapgChydrR92OsECYtLAf+lYrWkluulUi292N72+fbO9lnvLkweUBjqQ9AQVP7P6u8Zcy6Z0yJ2yb3inj7KndhXzNmxUV8uy3G9DTV9qrZ9J82OQhdxOmW2nu4d757/eNIjOCHIyZudg/1dks0VCt+XdwuFvfM98sPr88MDAnOuUOgdZTMEfrKBVbq6uspflfO2Myycn8qJqqpYKXzOG56R7Wa2BHGYwpbbSamrNptNWSOLhVomtYad7MDJRvNTERNUQUqMGt3M2taYeVQYsRzoML/sZPWgj2j5oaJ862QXTfYsKURkLDpmnaw0mrGqJ4c75JVj+xNRNln4gt1cwbLmxooHVpkYjEi7vEni02y+QYO5usMnaEhiZHaCOs/Z0KGXrA1VyeGU9EtBOoUjx+7bXpwfappBm09zuf0eYR5xgQazwKwhJd0ewzzkfW5y7/9yuTShKj/k3myDTsmCJlMi6sp+r9MzhkyZ6dMlZ1cT2/FifFxxA2a7wS65znLiZZNwi3ucmjlXpybrqJtkTK/52B+HCaAza2trQNrjnsm6u9Sjpj0ESTCLmHxopYhkqyDLZtbwvy2TWxfgq5gdxfVuTOaOGPMUItwCReiE7roKGTls0FHyBVmmACIZw4ojf+WhxL/UeqlYa5YqtaIiZXkXugMQSY5eMdces/unbk+YlXNh2bp/0o7w4TSDuxOT3twj/cm4X9HGxgAaMnFVTfJeb9wL7SHMBOZwpk1G4P08SBPhw4PQdrjlPQRhjw4fRBiXHGemxXyXpZMHe0QgQ0OLiw6gdPQI2KB402evj0/Pd9+ck/3d46OwNT6m4GYWXO6xwoCCSYF5Ch/CDElrKtmUdv89vaQyNUtcR+9kuaWbPtjdWE7BcOhQszTDsSf59262u1WQGXcgOaKWYbJ7JTmxJ/5yYsoMMZQJIWt8QNafGrbuj0HUeYgreibDR3fnZtekrnsEdnvjN1IogPvpO4QztLyfKU86BEsNfEu4weuWPdZ0zNz4DdPXLikQ8tjY7Vjsimw7Dr1Z32hHWbrtw8JQlAkD2yHrogKkEL6V3vQ5HWLD68r/KBt5U7gxUPjFiw3yGxG8rImOrlAZ+VrnG3l92pkOmXaAyB4AOeT/J8Hqixc/Q49Xp92WHKGn9EE+yV8O8yBglJIRnRfp+A+k/9KWwiSU+B6syS6HZVoMiakw9OZA8WG1c8mb0wMX18HpSGemQ0FEUaZBoXUDVk05ICheYH8XHCBwaON9QZ+I6h4T6f9akN5KpoMPJbjng/U40Q7Jvjl/CZ5VoARBbyVDEMuiG2FbQEQyJkh8YKbLksXBA5iweBEpoqnWZ77QPKXOefu9XTD4e+4VwASNIdPzLTAEwa+8IAC26rMTTEr2vQ0e9oAPQbLYmQnK6Ng6gHgWok0HoipIhCAZXJsWUQZObuAomMTdPdb3wUcfUBADpoDJPDEpt75zMbgMq0LQcYFmr0V+ErJCVwsIgcHdoS4TpAR5EZIq+XxB/K+g4H7GTINNsK4CUbUw04VD5rpI8MyzHaaIMsCc2Yd2WtGkThSSY3rFLcO+ygOJPegz9vaDGKMgIT8OqkAOTvwEhfXffMcE7gr0PZVIxdg2fHD28P05MM+GnbC+skkMCKh0bx/8xJaQzgehDdgl4BZanerD8jESFnWpFoQfYFwxwAjp3gdZkEtEeVWGu6hSNgy55o7sK821dXCUNYt5EG9cELCRK9t/yWMahy4fg4XRXEYdfTTD3UpE0zuOkYPmYLDjerNUQbAydMts9W3jhtgWwj0YHQiVMtiA+qZ35lHPdzsKhl4CSvn0u0smPkREutLOEsKhBggVxZNdYXZGVnEArRzZ4GmwwHKvc0OoNArbNzuWb5ptYYd16gora4nSLhGrAscuZUI7OpaU0peEnZt9Y11hpvKCGy+U3RE3DWVDkpYkQy86Iwzo04jcxmqUJRFJUhjgsHpk6H3TFQSCjAXLlW9KGon1l+ACDPWDFZbIJVbaZ2gMcn7iPyPBbc9zeN/3gBCH/qHxV2xraDLP1bjbN7RgzF4EYo5YA84kFbn4r/UdRi/k84fI2pNoVcBKJl8mD9mqaFS0Fq5MT02+QVauHmM3JBDWFzybPA9GHWodwVrWDhbtYMXyUXM+xJRNzNurEfXWf/FtDsGuIb0Kid2IRCkU0DVGlN4PJwfawfGuQlpCDgt5Nfilxq4npga2XiEvkG5eLID5QKWAUaWPYKzSXo3SQioWLNRKW9jbJWQC4cUpxbwqosQ0QeMWuBP88suoxnu6kHJIN5PUp6lw70ewUiR3lOtKo/MlYk10/t5EOh2sGaEGZjoQKGpyh+Bn3rMPEJLcBaHHPX0xeVKNbzsT+uy+GTndUdxAhJ1YYLpMHpiuFOMlaqeZLzGnpU//OROmKS+wTzHbFUYDUHO1kfickkxbWK4jCQ95pg9TTp5OWceYXiHPn5PlxSByXqWY6w+HKxTrU/cCPPCptFYQV8weRLUWiWzJsOUdBnLTWajjyibKV9lYPoHD1Wb6+8NaYJY+JDX+A5r2e3GOPAr9dkH06P5+iUO4AtF7ozfmrj7nD4JA0fFSqIeVUawd5dfcPu44ttRisYiloUysqADjIYxhTrYbpY0hYfqq4KuGLiFzgMCo3A2B70+/I+RtMGLSNJQXSsrmEoS1ke242sC2PWwzGNs7CAKhKS3EXGLiAJphsxw39LQgZAk6JtQdXNRSNmQywapiwcQbUgiHMNJwob417G4TcC08LjF55nkM3iHUv7R9F+P/S/YraUH7snBmC4ecEQmqKyD+Z8Cd58A/I2hdNiLskW2BTLgDWgfsM2YppLtFZWCczc/sGYu3bJIGPHuBIwytuBNqdU+ZZ4PXwxxCfdyiHCNM8emjDqND6CXTCcTj0BPH4p9+d6Z+dD6fhz4gga0ChX+egaEW8i10mIWCQmhwwJwxulZCCf4RiQzxwUj6CtAGW6EBRWBACdlm44l3o8SJ/XMfYvAh7VOHe3ncB9q8JGq+nC/ly6RUVGt4aqFYJ8V6q9poVevk/QRap5bFSO96Qv4pGsWhAq2PY3a664Ka5UFNQn9ekwMDNjBIgPgdU9qZD5lkQce+mi+Wg9T5ojozzZSymAzWaUIN3OPLOXw48lqkOrlGClsSkEW2Q1UVJBnMSNMVkwOnmxzOvU8fMVlMNwWKMd+JxgkKxaZ0jIYWm95ILV7EZSxUouhdzMdAcYMs8aIlCkAJJyQDs5l7NzPtiCJGOLnjrQbFEyOR4CKZ041WnPiklXRMdsnMUqzImhQXvG8L8aytnTEH5ufO8ZuDV9v/Pn5JwqKBxOI1V/6ZYSShK2JmwjijOPBsAYxhvJHpY2iG53NWaSakruHGoO0yY9VmZn4yyQLxFznvgydHqkUw/5F+omi06sT0Dtc6seW4FtPhRH/iBTWPew6M9l7gVoBd4nggCHcyXWGzOKRNEJBmHklR/oSxMT0K1PRRyESibfCsqWVsm2Y2MrDKdDVpRdli9V9XN9oS24nDMBo1zfUgQzjqlzZ3YWFL5EKzfDwMrR08ahPTd/HfdDWTOySySayWH/KBQvq2A1a5oxQV0SOZDblKYJBltxN9ghednYqunwkmst3nVt+dtNNK67Zp0onLFsogVmB9Y74fY5tbrviY7UlYcWFfwgJ37c1tfQWXmUyfA7ykFidfRYCDKqAN0bu2HU9skMRxM4yewtIRohCW1oJN9XWDmSIBwYXYnkUQdDkYdS1HWrHyc9fvd0LSzx36q+RIebGOMrAHUSsbwtn38cAZGGNDwY2CIIu0iKJsvFCA1mTiaMBNrL7gba4qpEa10NPRmAX98h23A27Is/Le9PDas1KNG5CCB9imnvsaSYowH0oO+r1Z3FSUTXUzgE6nEoOM4kbM+Z8T8VyN9fD0YFy8C4O6gQMxjhYgfOAW4TkI3BACR8LB8zOdJMv4gCq2PuVJDB2Gb8vgqgXNpMfJgeqGTRD5I4LmaMONb4ld4GBHLR4ur0VwP7uk5joWwxAMuZ6hSKJgavoQyfWCm1JTNMH7+qyyiizsRucLOz0VHIIIEZmNKftRWgy7A2UZ25dM4LKxSnNh4fxEjtkSiEDPxJy3HWHrbGsXQtqLjiKUCNZT5sDAzs9asN+Eml4ne+7wxGKUJeLsSlpGEG/pMD/ABcbjgxB3MCdcCZKmUtjF/o1Gf5WGkuJZmY7Stz3PHitkJGxgRynPmVDBJm6mJw0oaMpYHuxRxroGagDzDKiKkZXWMB5BjG2HaXKpdKNAMfRcRfRNwK8f2dAennTEEIhbEz80uSNuGBiiyPbksvfTzwoRzUKCVlEK3SU1hFy0ooYFw0rLa9iTmeK9/yyvMODMNGSdGGe0VWr95rbUeuv5L77ttV3mcJD8r8wIFm+Z3HZbJTUoQlsqVqkFr2e97dPd10Gxo/aHqIKqRjSFF4BMTakVk5ngTkzAqRZMukGpD0phqZjRAEcdUUXhAo76qovnYoeADtDiOxr6TAxqsJhbENecWM80xJGMDlgYBtmR9Ma4oQ3kOvBPk0uZEkwYZVu2glhB1JDSTUuN67WA2FBAGCXF1unFO0n4A45qYIXcedftp6W5gYH9uVOp1RrtzJyFycRd3iwzsdiJMFpRGCNp56QpCyOmuMDDxg02MTmV1iO0TIGDitvFuDmF5JVNsb8MdmreF11XRImF3miUT4I9ELFF286KEye4xKR5qdKHyuYLYBfBOonBt+wxPHYw+7ktTkJazOyo2eA0IR/3rrMxkeyPh5GpNGTkmg1NWbaYDQ1ctowHHyP7mI3sI1o4jWpq7VqtoYXMCnOshAGCPBrOCMwjh4+fM6pDTNCOdO0zxQoLxySHMBfNEjxpNMSRwQz5KiEFWUz2VB5HggQNe5xNTioX90cRkgERJd3cAGjpyJEhofAlwKaJHijdA+XTf8eMOL7rspQ5G5aWHhi4s6RAjkBmkEl2eqd7+9u9ly/T6r0rNxuNSE66d61dwohAar7RyJWKxcpzOp60nYGHa4g2GHsd/B7Gs/K2PeDPSi8hAZ4v2CV8jr1r+Ozb9kVYB/wBy2GdRFJfNnXwrFR/tlt+tl0ag7NYFP2KyqxSBM2f2ymVVciaRMkGuMgdtVmqP0eWYVCiHDTtUGGXjjhzXKhkIFGTwoeFUJ1p4vt7RMhEo81pWxNwtndM2zegAPPg4xW9mWYiht2BGc/dKI36CL91IsHHMgSm1wmGZq5CCEzMVRDAxWzxd3vv9nu7r4+/P/v3/lyNQ+CICqcgCIvjv6LJ3ndIqPzTCH7GmgmPa9aY6TJRugrZMXWG3MpJb6VVm1y3484Dbv8njBSuV8SzLxi4IrRRHDT1Eq1RQ61VDcpqVbVSaRSLTaOi6hVweHBMPY5Lnlovqc1ypVSrInK6EE0VyJZvCvcoZWtbTrNA/9O2vuXpuzU80RijkVY7ZRdpulT+I5xW2RNRAm26Lm36dJ9Zkef2waQvMsyyslwDTR6MUJIzZGMhX9F2TRpn+2c7e+l8Yc4SrjA7yVNGRhVrEhryTYkRTVErxbGvpvsNMZgId9Xisp1xP8WK3w4HHMc4JOhF1hEh9MBjFohqJIj+MDd02E0S81FAnX7R9BFMGaV7jkiTgOnl/AD1QRVKlA8kucQUi2oCFE+ydzOB2Ohu/MEygxhvGGNJt61FZhmOL2yfPi5iSEJl7p04kmAqIgFzQkvzElO+1aY2Yf53U9Yn2t0kkjxpk1WpIcRAujPQ7q1pgcHpSttJ4vY1Tm+RcJmh3kmwPYMLdueVcRH3QjFd8JLDDpRQFGIxguERaxZ+eQTWq2jvJoXvGzDKd9MIy/r0UWipnCniOOMiRcXVeREniAfeiZNdICC/4Pcl6onNhhKslKogwsBNCPYyQweBCPcAuorSJJY/zpMGWdQXS3god+rN/hi/kIObpym9AdeHTBY17kIMe6emX0IkSb20ZkukSvTxoobxC2D+Hbt9IGnMzwPcSP70vxSU+7N1bWgUVj4zRoZE1ReqHwKsd9M/6n36OLTR1Vyk/4u1EFuP1BDt0Mnr/YPjs2P41RPzV3hri5gP9pq8u2mc4IhAhKZD4GWyL+5EyEVkkIRxrtADMWXkzmXvh/2z897R+f72wf7ZYY/kyOHxXu/0KHjpXeM2jYXfMsMHV6607meW2glMV1sAKHcSwJE9JsMwMLSHDp2M8AjpIjmcAmucxXgK9sQyoZ8jvOy4pzP11VJgtnbSuyA/JVf1nyGq+0bWWCnwvPyVC7CqWyyD4cDNroWbRESJKa0Iuyof2ySwdskS6/e7kLRJY4Pkg8n9GbsW6F/Yq1sZxEU0Uh3sR5P4aBLvxySGp6rmTKVAaDPTQwFoKcX+v8au2RhsI35BH7zj6Utyqz84nQLzKjhV01FK02NFMRJJsCAj0YLRtOTEDE61SdxU76M/abBcnzqiUW+0vLyNhwt24XNZUQ8COc3kfTQDSvfMn6CLt6xC+PWgaaUD/D4Sd0Xqspqu9IRjrcmEsE53L9iuwu+Ri5hxNKdG7JJZSvyIWiSaSr1aDAc9mSnkIPQ7mZvs+gG/dNhskZTOMnLCQP7+NXFhmYFlw5qrNdfRQzqkLrdS2XO554s2QKVgXfUtoT+h+ZVn9faifbzo+F7Mz8/EVTda3FNOfiQwmfn8VDQkuQGRoJDIio7KxXC7P2orpKaWaitshWCxB9wKQfLLt0JEiSVbIUH+H7gVgi1+W1sh2OMv3woRI7NwKwQs5sg27Gu+6lbInOP6dW2ETPuTsgWSlhlsftRTNz/KpUWbHwcMNz4cBn4AHjBObHC8zEPWtqlTa6WtjeU7FXfddsDhf9htB1YsN4xmUe/XGrSv1in8lI1aqVlvskq1qdL733aQKn3bbYdE7XvddlhgBFfcdljI1x23HRZw9SDbDrIPX/+2w7zZe9x2+JJth8+CGF8xsK8W6zFk/2WeCFP9NaH65dJXjOo3K7C2Qyzhkmj1k9H3nwXe1/8U8F5tPEL3D45TleoIVPVeHeyf9c7I8en56+O94x96Z39ZvKqKSHK9F4Orjnu7bw57ITp1DDKDKBoi9OeWQd1RG1LkSpUGUa2EkKcuyRFCPl0I74KQf3Y5uBsGHbPYEdQaM9sIQIOnngZA12MAtJixt4GfF1i8R6T5cQbf0wx+BHT/XoBuDaLcJYAuqs83jucmgt1b4bkJCl8PnltsroTnQrGHxHOB/GfwXCyxDM+V+X8kngstfmN4LvT4FngujswCPFfNkwNKxF+PcyHc5JfcYrhTb1LijT59pJY3cuzJ3wLrxdPmQUfxBLroanioXBxF90by8HnU6VlA+C4UJGpcbpTSUONK+Tao8bbf58y5B8R4tvj20d7p8dE+lJ+vsAtcQGDt8Tsfb0e1fGCcuW8wg9Wa1UaJldRqkzaK1VK1WVerapOqg8oD4Mxiqt0aZ47Xvl+cOd04r4ozL+LrrjhzOlcPgzOLPnz1OPMqtjgMSfZ88pY5fVh2LJ3iiaZHZPpBkOkvOF7XrCKtqY0kMVMqSZ471PD1rxzwLpdjeLdcZr4mtLtS/orR7goGGSlotzixrv5poDd4Hn/KifXyXxT0PpXfMPrqAe8aokvnr3vHeAFQb4qXPcJlXwB4p/kGIeB9pxX5QSHyB1qX7oa8T5eOUCjT9QNhdwh1UugL6xSd+y7f+tR3qultE/XPRN6lJfnKUfdHM/KIun+DqLvaeETdl6Pu8dD/dqh7nMLXg7qrxfpKp6iL9Qc9RV2sf+4UNZRYeopa5P+hp6iL9W8MdYce3+YUNZ5BSEfdS4+o+9eAute+RdQd1PJhUXe9YlQrDaM2KFf7zXqDGX21YVRrg2Kp2q9U1epDnO7GqXZr1D1e+55Pd6ca55VPdy/g686nu1O5eqDT3diHvxXqfoB/aN6hOgzTI97+iLd/E3h77a+Kt5e/Obz9r/sXYv4qiLtawfOpZyen++fkbHv/6DyGkT1ibX/YGfUUx+I2kH1sOX8E6+fA+trnwfq7/JGWBXB9+RGuf7RBX4MNesT7/2Z4f6m47M+mPOL9WgJ0uOUp+xiFJXj/ViH+5+TjV731RXQzEr3TGV5P8Pk7i/HSsthdsh5zPQ3/IP86fmxk1n7DP1iPN17gex59/7z46/2kS1S8+0LcrgGGyPHWs0c+LP6OLcIjvFIOpDkhnz6a7PLTR/I0G93/G9AZQGzuBjeSJDCVTHBjSlqzWys226fuFzcZpeG+grhkV1wBF93QHbvMSkob74SO3VmBj4uvq5h3fMB4w5SHdYC5ncpzM8A9OnTm3goC67vfH3MPL4aS/CVHSdxsMXc5hwBAxXoSXs7h4Y0STnATxlx5vOlktri4/cT28aJyIVq1oKZVtUCtZqtiWnpDCNrOlsY0JboMInRhtwqBGkfv0RwST2ty6ymRO73YS17uB27g3v5bMntpI160F6G70T2OmWhVCtI0R1y8h9eJS7h4roDJrQvN9SfyarrFtx0ObBE4C4RaPAGNerZ7iH8gDQaYmODgSK8KFmK5rEeNLmj1ivXBwon75aDhQDODAn1xu6WGipkNeMJrn1qFQj+49xJvMcqP5YWXBhuI6y7zAyfan5q7CxOYnb8ec47ZtUXcTsZ9nC+BhAJmrq6u8kPbHprYNPEohAFeR+tDuH3RfYkrOfEhDIku1ZBXPr4SFeJtr8UEYIK/go2Ft3Mme48NunzYz1vMi7qaz+exGdc2fWEFYXGBlie2L670AOticBpcFApFsyGfimRU6UJrgptITQQGHCgkySQ1shBoHWqkvO4xts8gLvnMdmdv/RRjFtxrCH5Q+jWlMyhzdqJmZ1edMAfvC5XbAtkxs/w95tg+dMXLdp9kCNkyeeq9ndnutq77jJsR3D1X+h9ZBDrxIs5+/HpVodVYGIpDu+FmIFgW0H8/G2ZJSoF0s1K68fFzZ9UXzQM0xsC9cPO6Pc52d0UKOfv0+6Ujm40YXbEBmKd6nusTUEjxXMgbMCL7EFSCN+ERCEVGtsmDO2NFDHibVvp+3tfNm5UaObixrdk2cFtBSH9mGBa3Gih/yoxPn9fpA1wAy4VzJK4Vr2RSrJJgb9Zap08DoeA4D8ScifS+T4E+9bXZm23jVaPrOsU0kvWXuzvc4p6GG9TgA6wVCpFdcZ4+jZZ5Ivjs28aN5BgvccWnJ5knmf8HNAdKIKKWAAA="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514917.3/warc/CC-MAIN-20240719170235-20240719200235-00017.warc.gz", "offset": 109644433, "length": 10232, "data": "H4sIAAAAAAAAAO1963bayNLob7NW3qHD/hzZJwEkccfGe2GbJM72bdvOXNcsViM1oImQGF3seGblAb4nOZPn8NOctzhV3ZKQQGB8S8iEXGzorq6urq6uayN+bJ3tFZS8/CzzI7zKXVyPWIM4zB3ZlsuCxn3qQaMqq6WcXM0p9Qul2lDURrH+SwBwxjTb0XMH+w2y7TtWw/cNvdHtsZqsl1hOl6tyrsTK3Vy91qvlmFJTmNqt1OReb+dZZs+2PGZ5uUNm9b1Bg1TKarU6bhYE0dHINDTqGbZVGHjeaIsM3b4HXc0JUn+kjmZYPXuClpLW7WklVs1VK0BQSe5qua5ckXMKrVXKWkXTmKLtBChgZs13HD65nViQKteZrNdy5WpXzZXUejVXK/W0XK2o16uU6tWyWg+RHJzmWroOxLkNosj1vFos5ZVKJa9GjKZOn3m592cHDYIrchuFQtfomobtDdgfPssPDdeF5eqs51BLY/meUzDg3cf8aDD6t3lpNqnvDWyn4zL2wtCbRVWuBqhPHduzNdsExOpUk2e6sN3FcK3GaMCc3Llv4A5fHJ539t624J8qd05PDn9WinK5cw5vy5UQEb02barn9o0+c70GcQdUaZwet892i6WLytvz979Uiz8cvy1W1PLbtz9e/Hz6DpqDsbumrX1Ijjx8U3pdPf7PL+fv20etd2dnrf+e/7dd/PnoP3vvjk7KP++H3NRhN4yewfSIgGm5+DjwhubLj0PzWeZZ5u3FxSmsUwGplcmzjM4l+LVjvCJKnbzzTS7NJJDjGnlzdPEsowUi53HUHvvoFRDlFtEG1HGZ1/S9Xq72LHNJnesGaYHQjLxc29Js3bD6MJxqA5ZDJA4yemS7Xg5atA9N+RUZOSz2ZlbXswz7ODJAahrknHqviKyQd9TCNchElhv8n6AVyIGp7A8GUHo6GB1d76Ls5E5OW3v7u00ghVVw7+IStUUC5M0LnyHuHPAhB3woExWYEPJhixzRj7lWnzWLsvgzMdvb0/P2+fnBflOuVqvlosIqKu12e4pcZHJNKdIi7ZZlVS5WtsiIeoNmYWL8sHsyolru8OTNwXFTZybzmB6jbeAL2qgFmqYarVtJ0jZJVIAUKTtuHbWbQcNsKKC/ItdLlZJSq5bTwfZbF63d1nn7IUSaFLZ6CBKCwnurBLrMuWROg9iqe2V42iB3al8xZ9dGebmEM/tTbs+hVyac2FBWWSB+DdJ1UnRpqVQv4XHIbD/fP9mD49gmKNLk9P3u4cEeyeYKhR+Le4XC/sU++entxdEhgTNTKLSPsxkCf7KomUAxXV1d5a+KedvpFy7OxEFTFBwUvs7rnp7dyWxz5HAELbeZMlap1+tiRBaBGia1+s1sz8lGJ0ziR0xCTIzqO5m17SHzKFeQOZBh47KZjZ9SGCjeNbOzjmuWFCI0Fh2yZlbozdjQ06Nd8sax/RGHTQJ/YNdXYNrcGHigmInOiFDNr0j8mE1PqDNXc4wRKqkYmt1gzAvWd+gl24Kh5GiM+jVHnUKRY3dtL04PNc1gzue53EGbMI+4gINZPeYgJs0ewjk0uoZpeP8vl0tjqvRT7n0LDJ8ANJkUYZcO2s223mfSxJouDXY1sh0vRseVocNp19mlobEcf/OKGJbhGdTMuRo1WVN5RYb0ozH0h2EDyMza2hqg9gzPZDt71KOm3QdOMIuYRt9KYcl2QcBm1vDvtmlYH8BfMZuS612bzB0w5kmEuwYSlwnNdSUycFivKeULAqYALBmCxRC/8gDxb6WqypW6WqrIkuDlQ/D2gCU5esVce8geH7s9YlbOpZb7+Kgd7sd1dMMdmfT6EfGPht1SZ6j3YCLTtiyWpL1aexTcfTgJzDFYZzQAb+dJpghfPAlux7C8p0Ds0f6TMOPSwJNpMd9l6ehBHxHo6KDGRQdOOGoEdFB86vO3J2cXe+8vyMHeyXE4mzGk4CYWXPBLCz0KKgXOKfzgakhoU0Gm0Pu/00sqWrPEdbRm1rA00we9G+sp6A7td6yO7tij/O9udme7IDoegHJALd1kj4pyZI/8+cikCWTIE0LWjB7ZeK7bmj8EVuchtmibDF+6u9d74IW4x6C3N/8ihQK4n75DDIaa9xZ40iQI1fMtDXduw7KHHQ07N//C9jXwhAns0NBtWuyKtByHXm9sbkVdmu2DYZBFQ892yAYfAC3E2E6f+oL2ceIN6f9Im3mTuzEA/PLlJvmLcFrW+EIXGIx0bRibeW28mCYZL4CIFQA6pP9XTurLl7/BihfHvSUoQk/pk3glfjnMg6BRcIYvnrfjf+D+a1swk1Die2CTXQPMNN8SU+LeHAg+WDuXvD87dNEOjnc6M94KwkFZB4A2ILihYkOQvUD+HjhA5+AGxNaCPhHVPMbb/z2jvZFsBx+KU2/0NuJImyT7/uI1eFaBEASrFQRBPItuhG0BEkEYR/GJmS5LgoMHMGJxEMGisdRn7qieUs+8/btd0I3fDa8AKmgInZ5vgSIIfuU5AtBVtx4wwdnfbfCwe0YfOIuLGSGPTqxDiEchXnQgqoJGCHLBtWkQqefkeo6ETYa7z7o++Og9CmzAFlCZpyY1rHfgqY2HQtDxAdVeg/zKeYWuFiAChbtLXcZRcfQoAA0pny/wfxIy7jfs1NkIx0oQFXM1XThirosIzz3bYRKHAeLMLszTiA51Akjs6ZVh6fZVHlDsw5pxtZ/4HgUN+WEwBHrw4CcwbPzlOyZQV6C/U5GsgPDHB2cP378A4lm/GY6XXhEdAirNOwA/scG584lLAy4JqIVZx/Iwf4+4Rp0rBeEPUK4YYIR4HwMt8CXCvCjBOyhSNmx5xx3YVx3X1sBR7ljMg3jjAwEdubD+FzSmUegaQ9AwHZdRRxtMULcQ0vSFY+TQcTDYcb1JrMBYEbpltru2fk1sC9M1GB1wkdJZj/qmd+5Rz3ebEoZePJVy87dLRj5ERJq0lSXEgBHAVGRPdoHTGWnFHsxybIOnwQLNvWHoXKSR2b7ZtHzT3OJ6WKMu17IWh3YJtwoGLikT6tGhwJRuEnavD/QNiZnSS0N/Ke0NDFOXNgVqgTL0ojNcgT6P0G0uhlkgESi5Ag6HR4reN12OIOiYYa58U+BI2F+CBhjGBxaWCBMr9DNMBj2/Gr8hwpbnOUbX9wCRAetD5S/ZVt9kntsx3K7eCfbsZcDmiDSgTGARxn+t6zD6Qbz+FGl7ElkFHGQa8/ghZuWT8tlCy/TcNDbJwsNj5IYIwvGcZtPIg1KHUcdgy7YCox1YLB8l51NM2Pi5vRpQb+MP3zYg2NWFVyFyN7xRMAVkjRGp/dPpYefwZE8iDc6HmbTqxmWHfRyZHdD1EnmJePPcAOYDkQJCpS4mU6WtxTDNxGKBoZa2uL6dgyZgXhxTzKsiUkwSOoYF7oRxeTes8ZXOxBzizSTlaczcx2GsYMkD+brQ7tyFrYnFPxpLx5s1wdRATQcMRUluEvyZ9+xDTEnuAdPjnj4/PKnKdysT+uy+GTndUdxAuJ6YobpMI1BdKcqLj05TX/xMC5/+NhXWkV7immK6K4wGYORiO3GbkIxnmC8jCQ95Yg1jSp6PSceYXiIvXpD5YBA5LwLm+v3+AmBd6n4AD3zMrQXYFdMH0ahZLJuzbXmHAd80Fsq49Ar5K23OP8ChtRn//rQWqKVPSYn/hKr9UZwjj8K6XWA9ur93cQgXQPpo+IaGq035g8BQdLwk6uFgZGtT+jN3gEXHhiLLMkIDTAyUJ+MhjGFOdidqG0LD+K2EbzvoEjIHEAyKO2Hi++ZvTHnrjJg0LcsLkGK6BOLOwHbcTs+2PZwz2NsHMAJTU50w5xJjB+AMpzWwoNcJQpZgYVzcwUVVsyGRCVIlCw5en0I4hJGGC+Ot/k6LgGvhGSInzzyPwXsI9S9t38X4/5L9SRowvwDObOOWMyKS6hKwfx2o8xz4rwezi0m4PrIt4InhgNQB+YxZEtnZpiIwzuYnysb8XTaJA6tUgSMMs7gjau2cMc8Gr4c5hPpYohximuLmswa7Q+gl0wjE47ASxzJu/nbGfnQ+n4c1IILtAoX/no6hFtLNZZiFjMLUYI85Q3StuBD8K2IZ5gcj7kuAG3RFBzACAVJINhuOvGspjux/DiAG79MudQwvj3WgV5dEyRfzar5IVFmp4M0FuUrkaqNca5Sr5PcRzE4ti5H2xxH5Hz4pbhVIfTxnp7kuiFkexCT05ztiY0AHBg0Qv2PLVuZTJgno2FfTYDlonQbVmGmmwGIzaKcR1bHGl3OM/sBrkPLoI2LYFglZJDsUVY6SwYk0XX448LiJ7dy/+YzN/LhJAMZ8J9onAIod6RiOTux4I7Y4iMtYKETRe34eA8ENuvibTgIAIJwQDZxmw7uemIeD6OHhjs8agCd2IkFFsmcnsjjxQyvwmOySmWoMZE2wC963OHvgt8k+UksH+fzPyc9nbRKCBhyLj1z4zwQhCVnhJxP2GdmBdwtgD+OTjF+Gani6Z5FpQuwdLAzaLtMXnWbiTyYJEH8jzn3wyhFiEZx/xJ8AjaxOTO7Q1vGS41pMhhPriQN2PMNzYLf3A7cC9JKBl4KwkulynWVA2wgT0swjKcKfUDamRwGbNgiJSMwNnjVIRMs0s5GClcbWpBF1c+u/oWxuidxOPA3Toaa5EXRwR/3SNlwwbIlemNYY9kNtBy87I9N38f/YmokKiZgSh+X7Rk8iXdsBrdyUZImvSHRDrxQoZLHsxJrgjcbO+NLPORHZnRdW1x1tpUFrtmnSkctm8iAGsLE5vY6hbVgu/zG5knDgzLWEAA9dzX19BZeZTJtKeAkpTr7lAQ6KQKeP3rXteLxAEs+bYfQUQkcZhRC6ExTVN3Rm8gZMLsRqFkHQ5WDUNT/TioNfuH63GaJ+4dA/BUXSyw3kgd2LZtnkzr6Pd85AGesSFgqCLtIgkrT5UgJco5HTAWpi4zltU0OhNRqFnk6HWbAu33Gb4IasF/fH99fW1YqhQwveYRt77mskycJ8yDlY9yv5lSS9Ul4FqdMxx6BD3ow5/1MsnhqxEd4gjLN3ZlDXcyDG6QQZPnCL8B4EFoTAkXDw/kwzSTK+QBHbGNPEtw7Dt3npqhnTpMfJgeiGUxDxhwfNUcHN2OZV4KCiFg+X16J0P7uk5gaCYQiGVE9gJFEwNX4R8fWDYQpJ6XDaNyaFlXfhMpp3XPSYcZhEiNBsjsmP2mK5OxCWoX3JeF42NmgqLJw+yDFdAhHoOT/ztsN1nW3tQUj7oSlxIQJ7yhzY2OlTC/qbUNNrZi8cI2GMsoTfXUnrCOItDc4HuMB4fRDiDuaEliCpKrle7F536J9CUVK8K9OUurbn2UOJDLgObErFKRXKycRielKBgqQMxcUeaah1QAzgnAFWvrNCG8YjiKHtsI4wlW4UKIaeK4++Cfj1Axvmw5uOGAIZ1sgPVe7A0HUMUcR8wuz9+ptE+LTQ0ClJhZ05IzhfOnIHAcNB80fYownw9n/nD+gZzNTFmBhltKE2/nIbSrXx4g/f9rZc5hjA+T+ZHhhv0bzlNlQlAKENBYdUgrfn7dbZ3tsA7HjrUzRAUSKc3AtAosbY5GQnuBMjcKo5kW4A9UkqzGUzKuBoIQoHLuCuL2o8ZzsEtIca3+mgz8RgBIu5BXHJia2sg3kkvQkahkF3xL0hFrQBXRP+d4Qpk4IDI7XELJgriCaSdtJa43LNU2zIIIySYnZ6diUJ/4CjGmghd9p1+3Vub6Bgf2uWSvX6VmZKw2TiLm+WmQh2ypVWFMYI3DmhysKIKc7wcHKdjUyDCu0RaqbAQcVyMRanEL30iteXQU9N+6IbEoeY6Y1G/SSogfAS7VaW3zhBE5PmpQofKpsvgF4E7cQ337KH8LKJ3S9sfhPSYmZTyQa3CY1h+2M2xpKDYT9SlbqIXLOhKsvK2VDBZYt48THSj9lIP6KG69COUvmoVFBDZrk6lsIAQVzuZgTOkWMMXzCqQUywFcnaLWCFmXuSwzQXzRK8adTHncEO8VakFASYWKm4jgQNHVxxNnmoXKyPYkoGWJR0c4NES1PsDAmZLxJsHb4CaWfXpIbLyCnFa5YkRw6lgT0cMrwaako28y8dlnKUQyTCMQMvlxTIoe2Dpdo7eXPcvkgb8kuxXqtFnNO8j51L2CNozddqOVWWSy/ocLTl9Dy0Kp3e0GvipzPWiy27Z6yrr6EBXn9gl/Bz6H2En13b/hCOAQ/Bclgz0dQVU4kVrquyWCO8yMH/w3W1ylcKr5kHP6CjKtYbIXjweFSlbrNUVaF/FDXjJwyaSr1ceYGLhQ2OetBMwIBTcEVs1x4NjDGuETjmR4blG964CbPcTdAJhhu1UR8TdE2xC7FWnvJr8i2ahD1qnx+3zvangN8xak3CHrZevz9qLYb3zcnh/lHr+Hga2tcMNoX65GzvYLd1uBgZ7dft3R/O2lPAb0EKjElonpKZAo1SNtN0v3970D5bDPeb1vH+weFha28K/Ij6Dpw+2Hl9aorz05P22dnB1JiLwfpecb1V5wZtctDZ+/PzNLqQQRaTorOf+BVp365DQm00TqlMmBfuAk9aF000Ct8tO6RO37Bywn1sVEYft+LeHN7HSFgNdCCIZ39g4BuqvW5FL7OqXNJZrUbrtKIWFaYWq5TCjzr4pngwPAN9EKWqKiW5rpRrmMqemd7mqUbf5P5qyl0DofcC9ZN2F0Fch1zDK6YxHGmjU8p6Y9/lX6FWy55yCDSymjCy48K/JD5IATZ2lqUUg4VTYhrBDiUpQzJm0hXVz9IoOzjf3U+nC3vmUIXdSZoyIsxbE7k63xRJu3EaUXLsq3EBKJa3wzJnnLcT8QB3wbbCDcc9DhF6kbnCmkYQwvAUd8SIbj/Xd9h1MgkngTj90dEGcGSknQtM/fG6iTgfID4oQgn4gJN3t40cGy9eJKm+HkEM+zCywR3AXHwYCwv3ukEm1xF3QG4+zyJIpDTdB1Ekkt6YsZniZZo3P/0BRKVSUyWyk3QY6M4rIjCTLbIoomJNgVBtB1UgCezY/TBVKvWIImHk7oenCPoJEXETR0ITeD9ctVK0tsA43pMmuVoFRNx+kdB03hMVpt3IzkS5456bJ5dLEVmB0b0fJrVaRKoCi4uHJTLL95erMmC8GNx85raYBBb73kKhBjtpMRLY8TiqWaeV6cqDTmpbNzil00pvFuFcAboQHkesqKM8cycTDrzwQKN6bQrJ12D3H6ZdLOvmM99EoYz5FeZZSg+96FmUYA3gQZTsAQImUpZ3UHU4baRV6kVg3h4FR97BEjg5s6+pP7QtzkIyi3LX7z6Y+HPbd3NatIJZDJy5jICG8WHFExYLTPgCLH+YJ8qsZVg8/HnQKg6G+MlBvOWRsgUQV5HRTB4afz7MXXhtO0PqpUyrqqRMtOGsifGTqv4Dl30ocEyfW7zxcvN/wSsht461YVLwCM0YGhINn3lmsBL0sENDvZvPfRvj2DvLHJ89UuqlGkjc7sHJm7PWKdgGrnR4FLPY+BIePRh4eHJ+MjF+xuKDorr3MInlFBH95rNmDIH7d2ZCSMVYAaOVLNE3QmVwLO2fDs4v2scXB63Dg/OjNrimRyf77bPj4I1wWG9xTUegkmyeGH7Qeo/tIRk59siZudLAfRbOdIyaoMqfCQMFHqbGQ4VxsJNSONhayD0nvybd4t9IYWEfYOWkrpzUlZM6RiXOd178ygVVnvv4kaGiGDuT+KCLciVtAm7jxcstEljeJMTG3VyYNGcsQDjh3WwRZZPkA5Nxi7UNdF64snuZ6Vk4UtMhK0O9MtRf2FCHt5enDDivhGbGl+/QfvN7dh32kQ3BYuODcCAYHb9JXqkLboEy0wxurzYldXx9N4YimQPOiCTwYAw5MoPb46I+qXUxhtNZrksdPqk3mA9v4yW+Pfg5D9TTba1jGl0gF6DP/RFGKPMGhB/DHQ86xM/9Gi5vnTfSFbFbbDbREI7Z2Q+uheDzWnjObzAlQOySWVL8KnjEmlK5Vgr3PtnJ+cAFOdmbXPqhwfOfSZCUxYKHxoD//kfigj06oo41NWpqoUe0T13DSiXPNTyfzwEiBW6db3H5CfW0uBO/H92Xia7Jx6LtTFx0I5cz5YZlItU+3Z+a5E4W+hMYEl3RlfRYOeZLXTmoV0ulBa4cINgTXjlA9POvHHCIOVcOgv4veOUAZ/y+rhzgiu9+5YDvzIwrBzf/6+F9WaJLA8P1bMMJP9ozYpaLuT8X4xJ8iN8f/qKXD6ac/eW5f4Al3VqdLxmLwXhvIFg2rw3j1QGK9wNg8aL6i01xFkzcR3hEfMH9BCX1fkK1OOt+QvixLI6+Bhj5xEiNoQdTuomLC29A+YDwO/oidxfuWK6/aF2cTN8xOGOWoOShdXEU5Ketiyu1ilKhtFjXlJquF+WSprGKVtG1EusqavcJ6uLicN63Lp4Y/ah18RnqfMG6+Ey6HlgXn0HVk9TFxRqWvS5+LwW+qpDfoUL+mCmsUpWnH0Ej3nwmXFsKPKfOzecehZ1yXhHmafklLkKW6rEaZGRMlqkMWS0ucxmyinnCyQ9Tu6B7YInu3FLkk9fwlFUN759bw5uT2ltofBXHn+8dtI/3vtm0oMoLCKrciqUFgyWd58lFe+/t8cF/37fPMR2ICWh+Ms/RgOKzpl5YOnUHW2ED1rZe28a0TV2omJbqXQTFtDvZ9AeU1b6GZXtY5WJsfEJZSFggrF1AqJRWu1DuX7u4m/pe1SlWymj5lNGqYPDPKhjUa/V5BQMuVd95wSCRg7hXwSCBYYkKBtXyQgWDavlJCwZgCW8pGADE3IIB7/+iBYNq+XsrGFTL9ykYoJeTXjBoC9fUYlcenFGGXwrw7ZcF2mEGP76siVT/LTAifV8sF9PS95XaUqbvH56Pr5afNh+vqXJJVnus2yvWVVZktXqlrJbrellRtHq1XH+KfDzK/v3z8bHRj5yPT9WWC+fjZ9D14Hx8KlVPlI/HNSx7Pn6Oflxl3R81677KkT8gR16prXLk98mRg4Ff5cjvlCNvgRX6NjLkZf4hg9O3P59j3oVsBMmYzW82wYS+9I4q7y6SYLrm36KQJy3w0VLqxAvmtFMsdJDTTrGLT5q5fvI8M/j0KVNw/fCPzzOHZ3rJs8yrA33rgV7lhf9heeG6Wp6bF95d5YXjsfA988IxDEuTF67Wy8UF8sII9oR5YUQ/Py/MIebkhYP+L5gXxhm/r7wwrvjueWG+MzPywq8NdE/+8CkDIQI/xfIYDwP9IUxNmOvhB55c0+5jXIgqwSF96luGCSfaxT4Y4hojvNfACPXQ3xFQdGTDKLCrmu/yby12aDSSDRfMPbsUCCKt4/P24dFSZZ4529ZVOWQcJn7HrBNpYFUG9mFaOWIg3itH5sDvMRMFBEcQMhLecFbG4OPshLeCoTE8bDiR915qCkXWvV5NS7oX5bsn3fnp43RGnz/Fh1cnMvDv8tD/g2NYi2TgA5GbysFzgfzC6Xo8vk+brqdat6dUZKWo1nq9KlO0WrmowrRqsVqWqS4/frpeqKT7pusTox81XT/DiC2Yrp9J1wPT9TOoepJ0vVjDsqfrv7LZWpUE7vKouqJcwYA+YczJhiIXizlFkeubj/+IiguH6r625NWGcjVWbXiXJ2iblqnWUJSXudZQKaYm2oQnQJJ+wFctPdSrX6XyIH/9usMhCJh1v+oDH/oN1B5KaiV5I5YctfcP2j+0Dr/Z+63FKj5lRinR81i68qj15uD84kW779BLtnXWju6xnpztn7XPyVn78ODNQfv9TyRHzrme32WW/cLQDEfb8u5Zl0h1RYK6xFdyAB7yMKyvaQYfVlYZW6pQumLmCosqELOlTICaL6qpyPetqNxV0T+8wHJnvTWBSSivJS+yrDTXgzTXqgDzzyrA1ORKfU4BRgjVd16BSaQ37lWBSWBYmgpMpV6tLVCBQbAnrMAg+vkVGA4xpwIT9H/BCgzO+H1VYHDFd6/A8J2ZUYE5wC+8RjeNe6qUcC9WfHT01KSebeVJG0DAcqKji0pjn7kadbx/wg3++OLDp+4EDAifxBMwAV6N2YBP4/Gx6BCxYqL68QR4Rc1CVeupnxRQZxUtDnldAr/O3FjeTwSgfD5tiaGqdGuVKmNVpaZRuaTWa5UKK5bkcp3qNZ0+wScCxJm7b4khMfpRSwwztPSCJYaZdD2wxDCDqicpMYg1LHuJ4RH08qpMsPrkwLJk8yvqUmfzMRkBppIIQxn7Zgy5+tUy92Drv0bqXql9sx8auPXBOkuaOOKPlS/GnzrNE2OkdXzBryDnApVPNsLIQlyLB62/uTSPnuYUFiIjRM6YFVJ718x7qoUOMu8PsIvf+CcH1LQpuJIIs9zB0b1fnntaBW6h+lveB9OsDvOSHOZVMvqflYyuyrXqnGR0cfVU+UQYfK9UdALD0qSi69XqIl9kj2CPkYoGaDpsSnS92FIU+LFe3XXhl4ovVRVvCMPLXQPe4ozwEnuViuju+ZYQDs/uwKkxBTCHkAUE7fWCNY/7akHX7yBTPOOGHZyA0piAcAoE6TgMmDSJphRMkV9XX0f2YL34GizCenFfQHcCz2VdrcBSivvrqgL/cFUK/6dOLGg8mwiIxxDFyrzZxp7KrTNVJ2diIvaMTVWfN1XCb7ltNrU0OZvQ/b5l9CD4iHOzNG/S5KiF1lmenJlnQ2OrLM+bkAMvtEJ5cp7x1zXGJpu7utAJu3Wu4tSaxJcjTU1Xnbu28Tcq3crF2uSMoa8Tm6w2b7K4b3T3UzC0Pc2MbVpl7lxDG4QEjLlvenCqK0Mw79D6gV1f2Q5MVvFhVzt/+My5joiACUIqKh7twxz79od5UsRvMCwqRdF1h7tLK/h5hv1EymZqU7s+yJBnxCSopM6bL4RfeMYppaPZlsZG3qIiG4AvxMp6iiodcLd5URUXDbhtvvXqPmKrRgYFXeWxxZJDkpQIAAwVBJ5WGlApAgo8PNsyr1Pg1AjOsgNQAdVNn9KywTWz8AM38+gCt6IDxjLGR9FXGrcEMsrLKTFWBvRotC5Xunq51qvB/wqlvaJelDWlzipdCi+n7G9YbRnTNa66CN5KJK1S3aG/04/Bg+TqQbn6lTcw3HyfeS3Pc4wu2M4NiXsW0ubmFpn1kLn6LQ+Zq6eWskXx4WkeMldfvJT9D6hk44Lv9Yy5+qxK9r5PhjacZ+i1XWJK4DFc4vcE9gzL+PZL1fv+uirz9eGn22CFWFJeV6vBKuEdrnOiDn3HQYbbtSAGgHXk5Cr8U1R4XZl6Wl0ptQZdr8yqQV8w8wt+l8zZ2fRXyVDDdenMKnVwqBd4bl39aarUYxpE5BfkmrKTuaakqcdsEz9F4czJeXYguh5S07A+YCpqHMZ9/VjyC1+vKi52var4ONerFo9pccZVTLuKaVcx7SqmXcW0q5h2FdMuVUwrlyplqlRkrd4tlbvlGpXLrKTXqjqtyqUuqz5+TIsOwf1jWj563vXs4ozr2U8W0+KM31VMiwu+x+3s4uzb2Yc09hEzfsnhHdXsLtm12WD8fLZvN6o9pImnpkS3o/kq4bdY50RUe9dBImwtq+W0sFWt3+l5LzO/q3VMzv0f+3Lfa9YLBrAoaMsWwPIjs5wB7OzA8cvSUaosFMAi2JcNYHHGVQC7CmBXAewqgF0FsKsAdhXALlUAWy2XtGK5WiuVFaqr9ZqsML1eKxf1EoUftRiexwpg0SG4fwDLR88JYIP+LxjA4ozfVQCLC757AMs3ZrEAlnnEZHj5uGve/D1kwHPcAGoSZpEz33UBhPp4Gxskhug++engp5u/Id64+Vsz/5nhLvP4Z4PFQ067pgg28QzKIW8QCD9FLPiDjzr1eSTKv+cL+IRvsAV4JUbjx4gN8RK4dnvw/AVJCD7FDCrz4aE4C4rZhuXC0jgRPYdaHKZK+XNjOcyhINMyrL5D9UR0/haj8z28FY8fXvv6ETqepGWL0LlOWM4IfXZk/IXpqCvqIhE6gH3hCL2uTLryqwh9FaGvIvRVhL6K0FcR+ipC/7oRelmt1Cu03AUrrWkVrVquKLUqK5fkkszgb+0JInSY6gEROo6eF6GL/i8ZocOM31eEDgu+R4SOGzMjQr9wKH5GFGvLJiXOzWfuL/C3Ej6eFo46LnDB4HuXOr42wNHnpwfHJ7+0lin85ksV0WCieOuItsC9GkeWseVPRNGPgkkEwwposbTvIZn+8u8715ODLZiKV8Umcfq+dIALgrh0AS4eqSUNcGcGll+4FK4Wy4vcoQawL3yHWkVvchXgrgLcVYC7CnBXAe4qwF0FuEsU4PY0XdVqWrnO6nW9qvdKtXKV6TWVKWWN1kqxHXq0O9TgEDzgDjWOnneHWvR/yTvUMON3FeDigu9xhxo3ZkaAW86TMzjyDHYw5Rs7vvWS8nhps79/8g6QwV3pUnpMOv0RX4xJd23j8lEuOE+Cn75v772dAn8LvDByewPqmPw5lA+7Ew2Cs2wBKT8CyxmQzg4Etwv8Ecp/+LaHj3cMn3pl0csuf9LsgD+GS8NECD5o6xaCt5/ncpm1MJQD1eN6HXTzN/DHZmbtrwxMafQIf59Hoc3DsfIZ2SEKdEM/UEVN5ngb2WN/ePPZsfmjagEQbIM9IjefTXZ585k8z27y9Yzx9EDhuRuiNaHnoeHTrGm3F5y2S907Txm1oVGDpk+ZXA4fVRbwP/bw66zgNhikbUQuzImELyVCOScXec7lC6vrdPDJgpickl+YwQOjm1QiQwbAsKsj2/UksMMQNQwNsCUhgcltwn0MzZEUmSN+iPkjCtEYScIY8cZ0+JHDLifBsc2wwbjuZDhvlYKSNtQCuZocim3pE6HKmYTGNoQu4IKip5ZuFwI5jt5HT3vjr9bEmUj0ijPCNQHINj6Uguwf/ECGFNwLtDPANzg5DoG9jdSTaILtzETPTwzaOo7rot+RCfXdFADqCAgFR9w7mal80AZxlc+1LH8FOKrZnSNYIMiLS8ybz33xaE66Q8TzIqNJZ8x6xbqu4TGcGSYORDMA6PKbPB2UzGxA08DzRo1CoRvc8WFgkPJDw3UNfG4F3ujR4IQ4kQd0JLrwXL3mnUDsZFMKsWuzqB0Nu3hgAg4FxFxdXeX7tt03cWrigfZnXrPTNan1Yec1PnOS+BYjTmhRCb1kGnnDB8TnXosxgMceMFmw0ROrxwldo9/NW8yLlprP53Ea1zZ9L/giOZh5ZPsOpupBvegGDb6UDECzIZ2SIFTagdk4NZGYiNShEEOSSUpkIZA6lEiUQt4dSqKBT93NjvW6aBC7KbJ6+JTPlH3A53iGFjdgxEjJTj4fMezB0ob4SoUs+Jz+PnNs8AQwcfgsQ8i2aaSa0exOS9N8ZpjRVwVMQf8ri0+Id8D/68a/yo1LNQIDOMwb+pugWUD+/WzYJTAF3M0K7sb3z50UX1QPMBm7hB95zR5md/Z4Czm/+fvSEdNGhC44AZxTLW9oIxBI/rqQ12FHDoLLb2QPJME2DfH9dIR7X/eZpevnfc28XmiSw2vbmpwDv5KBc39iG2bPGgh/yolPP9fpGwzhlYtnJC4Vb0RTbBAnb1Jbpx8DLuB4DoR3E8p9FxxVRv2Omk08pjMxFL+2EQ6RUOYiV3+Lv2NYhtfBGAicgLVCIdIrzvPnkZ0nwsmy9WtB8cAb8m+/eJZ5lvn/ibEfz9gBAQA="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514450.42/warc/CC-MAIN-20240712161324-20240712191324-00031.warc.gz", "offset": 101668117, "length": 11074, "data": "H4sIAAAAAAAAAO1963LbONPmb7tq7gHR+yW0N9GB1Nm2/JYsKxNnfRrbmWNNqUASkjihSA0PdjxTuYDvNvbPTv7tNayvZu9iuwGSIiVKlo9xXtOT0YEE0I1Go9FPN0D91D7pFOVC6bvVn+BT/uxyzDaIw9yxbbksuLhLPbiolJRKvlTPy8qZ3NiQlY1y49egwAnTbEfP7+1ukC3fsTZ839A3qtUqbegyy1f0Wj1fYeV+vtGAJhp6pcpKiiI3y+Xt71Y7tuUxy8vvM2vgDTdIQykrlcllwRAdj01Do55hW8Wh5403ycgdeHCrNcXqT9TRDKtvT/HSoA0ViJbyaq0OvJT6jXxDVvr5ckVtVGXKmkyXt4MmgLLmOw4nbscbkZvNfqlR0kECSi1fUUq1vFqvKnlaa/Z1vVRugIDCRvaO821dB+bcDSKXmgWlXCnItVpBiQRNnQHz8h9O9jYI9sjdKBZVQzUN2xuyP31WGBmuC93VWd+hlsYKfadowLdPhfFw/G/z3GxR3xvaTs9l7JWht8qKHLR87NierdmmaBfGdvaGZ7pwvRx22BgPmZM/9Q0c5rP9017nXRv+KaXe8dH+L3K5VO2dwtdqLWyIXpo21fO7xoC53gZxh1TeqHR+Vd5Wfvjhx9Mf3v/y/U7noHP6o1L7+eyws1frfH9yHNTdMW3tY7Jm55f3P7bfVU/b78r1n05+7Oy0D36pH+6e7JyVD35oK+9DkeowJEbfYHrEwKxyfBp6I/P1p5H53ep3q+/Ozo6x/6C6JXL0P79bFYr81jHeEFkh732TKzUJ1LlOvj84m9Y8j33yitjoJtGG1HGZ1/I90OTvVn/Odxx6YYLkzmCE3D586FqarRvWYAPK+tZHpvPWLKYhbxvkI2PjPDWNc9DWH6lzuUHamsbGXlQPilNtyFAFPQcHamy7Xh6uaB9bpTdk7LDYl3m3vlvtfhoboHob5JR6b0hJJu+phTIokVJpg/8TPT0FBezY9kcD+nk8HB9c7qAC5o+O253dnZZGNVbDsY+r5SZhovHWmc/eEJjO730rD1KsEqW8UQ6luEkO6Kd8e8Ba5ZL4m6L27vi0e3q6t9uq6uV6v1RSdVUvN2oVrUZpv15vqDBhSyWlRDfJmHrDVnGq/kg9GlMtv3/0/d5hS2cm85ge423oY7/z0O+83KxH/ZaTvE0zFTSKnB22D7qt4ML8UsC/3KjJ1Vq91KinF9ttn7V32qfduzC5T2GoD0BDUPmv1d9T5pwzZ4PYintheNowf2xfMGfHRn05L8f1NtT0idqqTopBliv1Ok6n1a0Xu0eds1+OuwQnBDn+sLO/1yG5fLH4U7lTLO6e7ZKf350d7BOYc8Vi9zC3SuAvh2YIrNvFxUXholywnUHx7ERMVFnGSuHngu7pue3VLd44TGHLbaXUBSvcFDVyWGjDpNagles7uWh+SnyCStgSo/r26srWiHmUW8M86LBx3sppQR9xCYGK4lsrN2+y50gxasaiI9bKCeMbq3p8sEO+d2x/zMsmC39klxewPrqx4oF1Jzojwr6/IfFpNktQZ67mGGM0JLFmdoI6r9jAoedsE6qSg0nTb3nTKRw5tmp7cX6oaQY0X+Tze13CPOJCG8wCs4YtafYI5qGhGqbh/b98Pk2o0s/5D23QKVHQZFLUurTXbXX1AZOm+nRusIux7XgxPi4MHWa7zs4NjeX5lzfEsAzPoGbe1ajJWvIbMqKfjJE/Ci+AzqysrEDTnuGZbLtDPWraA5AEs4hpDKwUkWwVRdnVFfxvyzSsj+D0mC3J9S5N5g4Z8yTC/QuJ64TmuhIZOqzfkgpFUaYIIhnBiiPeClDi33IdfIKmUqmVJCHLu7TbB5Hk6QVz7RG7/9btMbPyLixb99+0w53Bnm64Y5Ne3mP745Fa6Y30PhAycVVN8l5v3EvbA5gJzDFYbzwEb+lBSIQfHqRtx7C8h2jYo4MHEca5gTPTYr7L0psHe0TgRg8tLjqAwtEjYIPipE/fHZ2cdT6ckb3O0WFIzRhRcDOLLvi1xT4FkwLzFF64GRLWVLAp7P4f9JyKqzniOlorZ1ia6YPdjd0p6g4d9Kye7tjjwh9ubnurKG7cockhtXST3WuTY3vsL25MmmoMZULIitEnay90W/NHIOoCAJSuyfCju3PZManrHoLdXv+bFIvgfvoOMRha3mvKkxbBUn3f4m7wmmWPehreXP8br6+cU2jIYyO3ZbEL0nYcerm2vhnd0mwfFoaSuNC3HbLGK8AVYmylkz6jAyS8Jv0Pab1gcjcGCr9+vU7+JpyXFd7RJSojX2vGekGbdKZFJh0gogfQHPL/G2f19evfocfLt70pOEJP6bP4JN4c5gHyFJLhnefX8X+Q/ltbCJNQ4nuwJrsGLNN8SEyJoTcHig+rnUs+nOy7uA5ORnp1MhSEF2U9KLSmw6opBgTFC+x3wAEChzbeF/SJqOYxfv3fc65vJK+DD8W5N/pr8UZbJPfh7C14VoESBL0VDAEoRjfCtqARwRhv4jMzXZYsDh7AmMWLCBFNtH71huYpdc7bf9hF3fjD8IpggkZw0/MtMATBW4E3ALbq2gkmJPuHDR523xiAZLEzY5TRkbUPeBbQpgOoCi4CSAbXZoNIfSffdyS8ZLi7TPXBR+9TEANeAZN5bFLDeu8iuAyrAuj4iGZvg/zGZYWuFjQEBneHuow3xZvnkFQqFIr8n4SC+x1v6myMdSVA1dxMFw+Y62KDp57tMImXAeZMFehsRJM6UUiM6YVh6fZFAZrYhT5jbz/zMQouFEZBFbiDEz/RwtrfvmMCd0X6BxURj5Gt++Ds4fdXwDwbtML60huiA6DSvD3wEze4dD5zbcAuAbdAdaIPi8eIW9SFWhC+gHFFgBG2ex/NglyilpdleBtVyoYh77lD+6Ln2ho4yj2LeYA3PhKwkUvbf8FjGoeuMQIL03MZdbThFHdLNZrecUQOPQfBjutNtwqCFdBtdUu19UtiWxjuQXTAVUpnfeqb3qlHPd9tSQi9eCjl6h+XjH1ARJq0mSPEgBogVBRPbonZGVnFPlA5tMHTYIHlXjN0rtIobN9sWb5pbnI7rFGXW1mLl3YJXxUM7NJqaEdHoqX0JWHnck9fk5gpvTb011JnaJi6tC6aFk2GXvQqN6AvoubWl2tZNCKa5AY4rB4Zet90eQPBjTnLlW+KNhLrL8EFGOoHKywRS6ywz0AM7vxm/I4Ntj3PMVTfg4YM6B8af8m2Bibz3J7hqnovGLPXgZgj1oAz0YpY/FdUh9GP4vPnyNqTaFXASqaxSB6CKifKqYUr0wvTWCdLV4+xGzYQ1uc8m0YBjDrUOoS1bDNYtIMVy0fN+RxTNj5vL4bUW/vTtw0Au7rwKkTshl8UQgFdY0Tq/ny839s/6khkg8thLq+6cd5jn8ZmD2y9RF5juwW+ABYClQJGJRWDsdLmci3NbcWChVra5PZ2QTOB8OItxbwqIsU0oWdY4E4Y5zdrNd7TuS2H7a4m9Wki3PsRrBDJHeW61OjcRKyJzt+bSCeDNSXUwEwHAkVNbhF8LXj2PoYkOyD0uKfPJ0+q8d1cDX1234yc7gg3EG4n5pgu0whMV4rx4rXTzBef08Knv86E9aTX2KeY7QrRANRcbiSuU5IJhcU6kvCQp/ow4eTFhHXE9BJ59YosLgbIeZlirj8YLFFMpe5H8MAn0lpCXDF7ENWaJ7IFw1ZwGMhNY6GOS29QvtL64gkcrjaT988rgVn6nNT4z2ja78U58ij02wXRo/t7E4dwiUbvrb2R4Woz/iAIFB0viXpYGcXakv7K72HmckMulUpYGsrEivJgPMAY5uS2o2sjuDD5KuHXHrqEzIEGhuXtMPB99Q+GvHVGTJoW5YWSglyi4d7Qdtxe37Y9pBmM7R0EgaGpXhhziYkD2gzJGpjQ6wWQJegYV3dwUZVcyGSCVcmCiTegAIcQabhQ3xpstwm4Fp4hYvLM8xh8B6h/bvsu4v9z9hfZAPqi8OoWDjkjIqgugfhfAneeA//rAXVBhNsj2wKZGA5oHbDPmCWR7S0qgHGuMJV75t9yyTbgsxc4wkDFHVNr+4R5Nng9zCHUxxTlCMMUV180GB1Cz5lGAI9DTxzLuPrHmfjRhUIB+oANbBUp/O/pCLWQb67DLBQUhgb7zBmha8WV4F+RyDA+GElfgrbBVvSgRWBACtlmo7F3KcUb+689wOADqlLH8AqYB3pzTuRCuaAUykQpyTXc/lCqk1J9o9rYqNbJH2OgTi2Lke6nMfkvThSHCrQ+HrPTXBfUrABqEvrzPTEwYAODC4Df8crm6ufVZEHHvpgtloers0U1ZpopZfEyWKcx1THHl3eMwdDbINXxJ2xhSwRkke1QVXmTDGak6fLJgdNNDOfu1Re8zKebBMWY70TjBIViUzrWRi82vbG1eBGXsVCJou98PgaKG9ziX3qJAlDCCZuB2Wx4l1N0eBE9nNxxqkHxxEgkuEje2Y5WnPikFe2Y7JyZSqzIihAXfG9z8aysvGeg67vd/aOTgy4JywXiildb+m+Ki4Si8GkJg4yywI0FMIBxIpOPoQ2evbMMmbD1HmYFbZfpy5KZ+ltNFoh/EZM++OQInQgmP7afKBotOTGlw4WO5xtXYgqc6E+8YM8zPAeGejfwKcAoGbitCNOYLjdYBlwbYzSaeSRF8xOWxvQotKYNQyYStMGtppbeNs1cZF2lyVKyEd3mS/+avL4pAjvxGEyPmuZacIN76ee24cKqlrgLZI3RIDR18LE3Nn0X/58sZSI9IkhitcLA6EtEtR0wyS2pJPEeidtwVwqsseh2ok/wRWMnvOunnInc9itLdcebaaU12zTp2GVzZRArsLY+24+RbVguf5nuSVhxbl/CAnftzW0dBZeZTJuJdgktTn7l6AZVoDdA19p2PJ4diQfNEDqFpaNwQli6F2TU13Rm8gsYWYglLALE5SDkWhxmxcqvXF9thU2/cuhfgiPp9RrKwO5HVNa5p+/jrjWwxLqEWYLgFtkgkrT+WoK2xmOnB9zE6nPeZqrC1agWujk9ZkG/fMdtgQ/ysrw72QH3UqkZOlwpK/LEa18hSQkWQsFBt9+U3kjSG/lNEDadCAxulNZjjv+MhGdqrIVbEOPSnQvo+g7gm14Q3QOXCPdAYDIInAgH9860kizjB9SwtQlPfOQQui0KVc0hk46RA80NSRDxxwFzlGwztngGOMimxaHyShTqZ+fUXMNiCL+Q66kWSQSkJh8iuX40TKEoPc772rSu8lvYjdYNOz0RHAYQombWJ+xH12JxO1CWkX3OeEw2VmkGEs7O45gpAfR5yqe87XBTZ1sdgLMfWxJXIlhOmQMDOztpwXwTanqt3JljJNaiHOH7VtJuBFhLg+kB7i9uHQTMwZxwIUhaSm4W1cse/UvYSYr7ZFqSanuePZLIkJvAllSesaCcTUykJ+0naMpIbOqRRloP1ADmGbTKR1YYwzh6GNkO64mV0o1AYui1cuRNwKcf2kAPdzki/DGssR9a3KGh6whPBD2x6v32u0Q4WbjQq0jF7QU1uFx6pR4WDCstrmGPp4p3f1hcoW8wUxd1YpzRDWXjb3dDrm+8+tO3vU2XOQZI/i+mB2u3uLzpbihyUIRuyFilFnw97bZPOu+CYoebn6MKshy1yZ0AZGrSWil5E7yJMTjUnEk3KPVZKi4UM9rfqCMyL1zEUV927ZzvD9A+Gnynhy4Tgxos5hXENSfWsx7GkPQWWBgGtyPpjTCZDc214P+eWMmkYMJIbUEF4wQRIWk77Wpcr3l4DQWECCm2TM/PIuEf+KmBFXJnPbffFt4NDOzvLblWqgebHhImZjXu8uaYycsdc7MVgRjRel4YsxAvxUUektfZ2DSosB+hbQo8VEwWY2qKty+94ellMFWz3uiaJIrMdUgnBUiQBOE52s0c33KC60yapyr8qFyhCMYRTBTXAMsewccW3n5l862QFjNbci7YTmiMup9ycbHsjQaRwdQFds2FBi1XyoVmLlfGrY+RlcxFVhLtXE/1zZ5c+yTX0FLmuFmWdnzTZJ5hie3hjMB8cozRK0Y1gAabkc5dU6w4d2TyGOqiOYK7jQY4PnhDfBVhBVFMdFZsSYILPd7pXHJ2uZgkxbgMiCnp7oYSF2G1HudZ2paLCskTuVkvizf8tg8rWBBhMaWrL+fUGhiAzsGbRX8IMLdHDmCk43QxfBv03u7jErRN1kJxYDsdOjSYA9jqv4PW1reKRsTfrJUIuRQuH/jPpEji0Dqtyq/lZqMRDYbmfeqdw8DD1UKjkVdKpcorOhpvOn0PF6xef+S18OTIy3Lb7hsvlbdwAT5/ZOfwOvI+wWtYvkB9DHa1Atqxyzx+1kLGpOnOiLdI5VWHhOM/wbLT85p7H9PTWhMXxbKZG1FnYFh5sXJv1MafNuMLKabBE7MVbTfx7I8MlmWlUmFqvUL1sqyoaqmq12W50exrfbVaapRoXSI6IADPQPOP2/IapXqzWscI4tyoIo/w+CZ3FVJSvMIQhOOTlgMW29BWcGtfrJHU6in5lMnC8a9w3HPHvATaN03Yt0nGVRI72MG6zTVRorZYEkwjGKUkb8jHfM6i1EUab3unO7vpnOGdRXzh/SRXq8LNXhGhEt8UMZNJFEdy7ItJ8D0WNsEUU0K+Uw4ZXwM3k2EMqW9beh4j0Roajd+s//t/iO2TcMr/Hmh7sOjHmAidc740IQzm82BL3QZcRnwragFsxFZR3SapngD6kMn9EXgCSi69chmCV1m6mwmcsU2PY1ywLI7Hdjz2HYiHv4qh2wpC4J4TsuJFnGNcP3DleZg30kh1kB847HJqEEHQf/a0ITAqbZ9hBIznDgRtmMs4nxPlA5W+h5UiToUH9pO9uRwDxrtbd2CZRC5CrCg0cYNM9y++MF99mceQiPi5d+JIBIQxoDEj43k6PnPCTyLbyUWPbr8h7SAcmc76JZj9u/FtWVdfuCzF8PONg/PEidowjxNrzHfr3YWXvREeF8HUXooYZaVMxvOIu4Cy7kT6LWAd6qWRbRJtNI8sHk7y79jpfdHG7NzEJOfV/6aGS66tawNRWInMWDMkqj6PeR7/uxPrHepdfRkAcebO05i5ys+ph7ov18ug+6e/HB4dn+398KF7GjOOc7gPciHe3RSOs0QAPGiACUx2416EXIQdqSoKdKQjI/9i2KLlpju12szp1xjK2hy6302rjHOw0/zM2Z/+3H4JYx0xEmRgVkMvgnuycT8i5gulhHUyJ+KpOBF3WLvJb8k183dAYveygolxLoi3vHhDDlOvo60XHzcJt8CFoH/XWMKETm2QW5nQeW2kemaZEc2MaKoRDbf8zBhXHkJcnSStcTrz/HSPfWIjMFR4elza7k6+JFPRwdYJZprBlg9A+pM9L7Emkgh+VUD44aTk2Ay2XImgnqaCLto6y6vU4US94eLyNia/O/C6qKgHHnrPNFRgF0qf+mP08BZVCM+uTCrt42EZw+VXF9V0xXMYYtTEhbDO9m6QT8FDzhwMDGd0h50zS4rvn4pEI9eUaiMc8+RdIQh5+may6yfs3GfTRVI6y8gxA/n7n4jrO2iRrZlaMx09oAPqGlYqd67h+ZwGqBSYdN/i+hNaU7GRbDdKNEV7y0jq1LJ1fb50ms9dOguVp1Z6luJZjdu9yJdM2daSDLHNFkiNbSXzK8kmEveibYCxYOyjpXrq9XpjmVQPlnvIVA+2f02qhxdZlOoJCjxmqgdJ3kuqh84kesINYdckeq4pdv+JHuzy8oke0UgrGJw5gGVfao9xIb0cuxwDoJteICcABK6+mOLMvThgzKCTo7EbbLo+N2zQY43hE0N0H50813DSsM5TSOCotv0xytYMmOWwVuKSKkjtv1TqE2G8VEo6vnCBwPvJy075Zbt5Lt6EaOAyCgfeTCzKBRTWMym8hGKCj8zDOz6+iCZAYBEHX50BQPBWq1kH+YKEK/lSo9Gs5OsT/ngQr6VUq0gxuoyJopZSAvQcANDoDu5xgAr7fFVBksY5oKjo9thXWx3m9CcX8IBGC6yc4d4k3TZT9uiw+337cG+m8J5L+bo2XaHT3t8/+rB7P43fOfmHE/Vhk3/9Ron1S3WA/qWqXparFapXlEa9zqq6UtKU2i2Sf7g7hMMozx5L12UChSm6dSYwUf1+M4FzVrBlM4FzObtrJnAOXw+TCRS9mJcJfCopqPtcs0L0vkfw6X10jMy4RCaynGWr7j1bRTaXjRlW5RLYou3QspLQ9t6uNaVeC7kKDL5o59i5+tKnmsGcN6AUWuG++Jtud96QMV2+03B1dYMLYnbyzOsFn0jukE1EA73gyy6MOa7I0dGrr55URL9ibo7K5lv57xBdhQYE4r2JriPZSANAcMK9IcK54WFJyx8ViFKupkcCHiEdCi7a10iHKjKpktX5GVF8vs/dOn26c1jcOz09LHbbh/NUZsaB/boJ2m8hw1pDPe4cnRx/OCXvj961Dw8xRxBLESzVjFJCO7Lf7Zx9OOmC5Sf7bbKzt7N/44ZqmKs46J69O9oV7Ygmv9WURRN601ZqsYxFwmTwyM+Uk7FcsjXVUQuSrY/jHj1YQvC5OglpCc9brOPhiIWL+RvCAXpK43ypCLOpoQFPFlm7/VK4icvgOilEWXq04JiXDxmcNdZ3SuU++TRsZmmfpqWNZ2SnohnJ3iWOpJ7jE4Y9j8Gl6fOo54bOCmNrEKH8/Vf/KpWbm6lWOYwWn4cfbmChwypgp3NTAxEe5cKzS8FH0Bse5ZNg4ikw7fisq0t4GrTHOccuDDD2A5+5nCd38ADqy3L7pfIW/olu5plfoCP6Fz4E2R5Fl+HDMS8mwz+kJz4VSo3Cwa/ir/DHePBKBLbQLWy9eDH58uLFK5AsEG4B4+JDLgjKhxKXkrqZ5c+/7fx5Qy4vyhDzOb0oB8o3Y/yH50ATwcXb5UATTTypHGhzyRxo84FzoM3rc6DN63KgzcfPgTafXw60ebscaDPLgWY50BvkQJtyvZSXZ3KgteeUA72PlGbzYVOatEL7NcbkMoBpSisNpaTKlbreb+iqputq4xFSms27pTSbD5fSTFuQbpDSTOfsHlKaaXw9WEqz+UxTmlNBOzBrSpbUfHZJzSz5+MySj7Wvl3ysZcnHieeYJR+zkPi3GRK/Pvk461I9ZPIxxZF5tunHx08T1h46TVhbMk0YmNUsTZjZxCxN+GhpQpx1cpYmzNKEXzlNWFl0UjJLE4ocX/PuacLm00sTKo3qMg/FxGIPmCTE5hfnCHmJBSnC4P4jZgiR4vNKEGKPb54f5CMzJz24y38MIHjChEveAzbx3W88z7eLS3ApyKYFPcM8W5AV892ptNwNyos0mazUZ9Nk+MiZeWmy98y3mDtJ+fHUXd82EvmytyZgqhixW6bM7prVQm152KSW2qC0Uq6US7Sk64qqan0mU11rVMqsWavWK/f/kE4xA26bxkrUvtcs1hybuWQSay5fd8xhzeHqQVJYog9PPYO10Epm6aYHf+LjE0jnyJVyLKET2Oqnk9PB1ecJ53QqlRIfdrEMBlgeFsBJaqf51RI7sJp/lYdsNrKHbD5CxBHVrnv64fTGsUFU2MOjDz922x/IWff0rH3QPTz7VkOD+PCD7dpPsdBgt3Oyx+Onp+29w7MuWeOLiLtO8gR/dN5IhOBcog2dENZ4BrNSFr+l0iupy32QXUldZB/10Yy3Wg2iKPxkSXjDnz+Z/pDHeuwhj2gBbpNsWGRNN0lzknG4ZR7hWTw5MrMMj2wZsqdD/meFrZVmSVkQtUadeuZB6wRAv1XMOtHCkwlZlxv1JSLWUOoBA9bQ+uJ4NRZYEK4Wtx8xWg0En1ewGjp881g1Dsu8kyzo5lgD/G1w4fG8tQ2iU8vFZ3v/N/6eLT7qG/PC/BojIxvWOvzFPp+ZS4a025233Z2nFMveF+Fi3u1E9Bg6j9+hq3jlpVLHaHWjGYghOg4S3McLXBzwLgQyfTTl4ekER0yU2s1i5/dxxIQP653OnXRPjj6cnoLH9/NM+X3bn6XYaf/a3d/vns6UfgdDbdw1Rg+z5GFD9HpFrdaUcr/W1Bt9taKVVbnSUGE10rQmY3X9/kP0fOLfNkIfr3yvAfr0VWLJ+Pw8ru4Ynk/n6UGi87wHTz04f6d1IcRP3U8AK6/+cXlBDUGmNsSvWZD/FkF+pQmgMlwrcVOaWFVvs3P0vvazyjVsh5tqEjPmt2QKZsU2N+QktPPZ8ZTnkspYcDylrny90ylK7aucTlFINUtjPHywsoLWq9M+63bedU+zfc4Pc/YjzdsJT348pI9xhxxHttI+9MmReZkcbnCjkyPCDN7jwZG6kmVxMsP4VQ6AZLmZbz83Uw5/szY7UDAnNxMPztwqNRNv4MlkZuRmbamfXYJi95GbgdJ01JLoy3JblvFwVH3HhTeFn5NSDB1eXpZ3DLwNFOEj3pVr4nbft4RqeDYsKaYpCvMSJVGC9vtBnyf3GsGtP0CjGAad8QZnoDJhICSBRXoOAyFNN1MJSBTw1FZoVF+W34JZfVneTf6I6kulBl0p7wYHunTxHmN3hpqIwkxKlGuLqE1cnmsp1acpMREJiJFqLiKV8Hyuo6ZUpqkJy+9bRh8AYVyalUVEk7WW6md1mjJPBsR6WV1EkBdeqoelaTpaFB+IEVvYu9CZu5ZWeaZPvppKrr6wb6LSUlJsTFMMHYYYscYiYnEH4+azYGR7mhkbtNpCWiMblASWct/0YFbXRrC4w9WP7PLCdoBYzYdR7f3pM+cyYgIIhFzUPDoAGrv2x0VaxJ2/ZbUo8hRvrq1jUED7gYzNzKCGPw0do6csoheWX5rijNHRbEtjY29ZlQ2KLyXKZoopHfKftl3WxEUVrqP3sr6LrdWjBUWjjjdZsUohS3JUABYqQOhWWqFKVCjw72zLvEwpp0TlLDsoKkqp6SQtGxwzyx8t5gvcih4sljE5inuVyZVAR3l+LybKgB+ZVWuyrjW1alXt0361VK2UG02qq1qlJlfZ7Pobpv9iK3yUBhSylUja1o0e/YN+4k8lBYcg2MDxxhsabmHAvLbnOYYKa+eaxD0LaX19k6Q/sBRrL3peqbg/u71DpMQe4nGlQHH5/R3/Ads7sMO3eFYpDsz8DR4jwzJc7+ofEUzDeO3kihv8bjvgyUPbP2fUJ2cMSGKnv/ETi3z3hegpP3TYmOysEGcRk/fwCpdFuOUikAd8iiSStr/joSiInR3Vipy6s6Myb2cHIBDbjHab7BrMT2zuOGW+Yd71NOR02Z2jw19mCh7DzZkNHEdn77qH3bOZwl3d9qmjzzJxuDdnt8fsQ0xPDve6J7NsGAim5+4NCUzXtU8lhRn2MJtDJjwIfBuEpXLTYamkQ8PDdGgrQspJOtvHzBlR07A+YthqAla/PmJ+XD4qyjLAHUo9Lm4Hghlsz2B7Btsz2J7B9gy2Z7D9ScF2vaaV6rJcLWvVcl1tNuu03q/W1YpOGzVaqsVavi/YDv7A7VE7Vl4A2sXtR8TsQPBZQXbo780RO47KXMDucldStc3o7O7VF9P4c7Ir4Rs8cIHYN+pV4sFAwS9z8P7N4OsbVpv+QQ4FZu/sD3JUqkoapm40n9RpibtCVlCxp4ZYcao8TcA6Fyg+8hnA+nKHAOv3cwpwecSKFDPImkHWDLJmkDWDrBlkzSDrk4KstWa9pjbUBqtrfao1+kq5We5rDY02dCZXgOt7h6zoENwes/Lai54jUJ/zIIEHQ61I8VnBVuzwLZ4kUF/wKAGKDiUANTzNMXCu/pfGvvUMMhXwM5ZQHTgceCoam8aqS5WNAdRmSW6U5LzSyJenAWoZHJs0gCrPAFREoCdgZxlMGxb9iKWrGcHPVzrMNAYGg8X3vnO+3bP23uyx/RN6iYd0ZhPEH87O0rKzBwbwPZMm3mmffOj8MFO4belCos0Zxj8c77c7szno1KcOfDg+OpzNQL+nWiKUEJTeB2tZOj46O+ue7M0KZm9gUU2M92NCe5yFTw3bc3vyNMH9fFT9yFnxWqNUW2YjOZZ75J3kSDID+BnAzwB+BvAzgJ8B/AzgPymA31ebSr/epA0AEzWqVqrlPq3o5bIiN6rNPq3fP8DnHsEd9pLz6os2kwcFHnM3OZJ8ViCf9/gW+8n52MyB+WKDMe4kd/CMtZdMUX/jiD/aPc03cQdoEzo5L/U8FQS4dfVYXKDRrCilfK3RzDdm4wL8MX+FZGAAxFF7yMz1TaICZ+8AJ7c/7Kfgayid/+WcP7fmbluxUTmfGvoVE+1pwt8FsPNxGanVq80l4C8We1z0ixQz8JuB3wz8ZuA3A78Z+M3A75MCv01VVuuVUr9ZaTb6Wl8uKapeqVVrFa1fU8u0dP/gFx2C22NfXnsB9A3uPyLyRYrPCvhih2+Oe/nAzIO9+NMsvsWIN7z6Ypv2wAifWgj2ynOWBb7iIXCgD+SgfXK2d/ikwK+NvloJOgmv3lDgVNHVxNPrRYenoe/tKk/v2K6W63K+NnMKWk59vn1zLu7tAF2PCujbN3Rm3Ar5iiGagbI4hrx/dfuefkV2SeiL+vnUkC+faU8T+M7Hm4+cf65Vl9nUXas+8p7uWjUDvRnozUBvBnoz0JuB3gz0PinQq1T6tFJtqnpT1foU0GtfrTeqTUVtVvp9gLP3D3rBH7jDjm6ovGhDN7/9mPu5a9VnBXihv7fYzQ2jMn8z9wn+VrDjBD/sofukMwSM5PFniEkseuw+Lgv8EcVL4V98KhTi5tMPJ9//8pTAL9+/fSJg66TbCF3x+Vyi69HDvV4qdfZJlBXp3IYrfqSNiyJtZ/h9tyxwsdyU03Bx7UFPMvOhS33cV+pG6O9PABHP5oPFc7lmf+Ht5KT76+w2ceo7hjbbdPfo5PtZqN32B74Ly+mjbsmuVZ8aMkeT8DSB+VxA/Nj7wstLAXMo9tinrcsZNM+geQbNM2ieQfMMmmfQ/GlB87peAmSuq029Uan1S+VSTa7ptUal0mhU1QZ9gAeEoUNwl9PW5cXgXNx/1NPW5WcGz6HDtzltXZ4L0Ns+AWwGJj7IQoujx9/49us2ImLRrUTSWHRuClcvWTiWYS7BP7lclRv5yjSSVirl1Azz7JHrR3zO9t3PCZefHirFmfBEYelcPPjIG7YrSqO8zDlhLPfI54SRZAZNM2iaQdMMmmbQNIOmGTR9UtC00Ww01XJFkas6gNSaLNf6DFFqra419Ypeu39oyj2CO5wT5tUXnRMOCjzmOWEk+azgKe/xLc4J87GZA1DlAtmHS76DT58i1LLQegS/NnUAY/uNQ9X9l0o97B0AwKB/sV94wj5OZ4JvVGd6a3Sj2ijnSzNbo+uVWeAK/X7QFPA9HwmernLcTv0pp33bN+5+fBh19qmBYjH/niYqXoBGH5mRel1WloHFWO6RYTGSzGBxBoszWJzB4gwWZ7A4g8VPChbXqNpoytVKs6apFa2qlJtMrrAmoF9drvS1B9hMzT2CO8BiXn0RLA4KPCYsRpLPCxZjj28Bi/nYzIHFSgaL7xkW46OymtOwuCan5HOh3415sNi1fRc9j6eAh++McVEBnxzG5ZPpiWLc+dByq6iatvbxT9/22PZqKGTJoucqdWCghjgWWxpYJebA/Ws43nqRz6+uhNAQzJjr9RA2rOHL+urK36tA0ugT/r2AqlyA+ekzsk1kuA33gStqMsdbyx36o6svjo37P7AgLDX2mFx9Mdn51RfyIrfO+zNppw/G010TVxOLBlz4PI/s1pJkVeremGR0DddIuPR5NZ8HCYXy56IOtZdLG1a3LWxcrE0SfpQI5ZJsSdMKOEGdfNeBIr+yVAdA9cDCcxOVVya6FeB8tKhERgzKwqCObdeTYFkHEDIyYFkK+UuOEg5juLJJ0crGp3B+4LBLXNcksa7xi+nlxw47ny6O1wywQ1CDi1YuymlVLVCr6ap4LZ0QGp3p0ngNSxexQ/jOrchWMVDj6HtoXMSnFTElEnfFFOGmAFSb9A1YlfZ+JCMKvgouViA3mDgOgaGNDJS4BKO5KpasybWe47row6yGFm+mANoIQJZj7unMtT64lvF1gNtY/gnaqOe2D6CDoC4uMa++DEChXbQ3BFTDHW9GROdQvWCqa3gMKQPhQDODAqqhmobdQ8XMBTwNPW+8USyKO96QAbYrjAzXBfI66zsUlrhC34mcqQNxC6fVW34TmJ2+lMLsyjxuxyMV50sgoYCZi4uLwsC2ByaSJh6Yf+a1eqpJrY/bb6nhMP6EEyf8QQlCz5lGvucV4rRXYgLgUAaIBQM91Xsk6BoDtWAxL+pqoVBAMq5t+twKmoYKlMf4gBWTErAuukG94dU/KLFCIRfyKQlGpW2gxrmJ1EQEI4UaktWkRhYDrUONRC3kt0NNBLDj2LmJWRcXxGiKKOHWsJw2DnB1NVxzA0GM5dx2bMbE76AP6ZucALiv/i5zbPAFMBD53SohW6aRuo7mttua5jPDxM5uFaHUTOl/QRnfc5hLQjXjUuNajYWhONANPVewLKD/fi68JVoKpJsT0o2PnzutvmgegBg7h5eCZo9y2x1+hZxe/YPXEowuSQDmqVYwtDEoJP9cLOgwInuWC8rie6QDmmDzZ6Wi+LnzdRsqql/wNfNyKSL7l7Y1TaPom0L6U8Mwn2qg/CkzPn1epw8wQDUX50hcK74Xl2KVOHvT1jp9GnAFx3kgnJtQ71XAr4z6PSXU4ZSqAN0A7lnCmIvo/zXujmEZXg/hFPgAK8ViZFecFy+iZZ4IH8vWLwXHQ29k4qfvVr9b/f8SsWIFdEQBAA=="} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514450.42/warc/CC-MAIN-20240712161324-20240712191324-00007.warc.gz", "offset": 6207916, "length": 7512, "data": "H4sIAAAAAAAAAO2d+3LaSJfA/w5VeYcOXyWyNwEkAQJhw5RvmXHWTry2M9eaolpSCzoWEqOLHc9UHmDfYv/byXP4afYt9nS3JG7CgAF/Uw6ZMRZS9+nTp0+fPr+WMD/tnR+UlKL8PPcTHBUub/ukgXwS9D03IPHJQxzCSVVWKwW5VlDUS6XeULWGXPs1LnBOTM+3CseHDbQb+W4jiqjV0Cq2YhqaVjBNxShUNEMp1C29UrC1sqlV9IqGddJ6njvw3JC4YeGEuJ2w20CVWr2qDU4LhXC/71ATh9RzS90w7O+gXtAJ4VJzTNWfsG9S1/bGdKnX7RqxK0ahVq7ZhYqC5UK9rFQKRq1s4LomE02vt2IR0LIZ+T5v3BsWUiXENBSNFBS9XitUTFMvYFw1C9jU1bpMlJqi6omQ47PCnmWBckEDKbJeVMuVoqJpRTU1NPY7JCx8PD9uINajRqlkUMOhXtglf0Sk2KNBAL21iO1j1yRF2y9RePe52O/2v3OunaYROQ4Jqdu2aNB38O0rajXLqhKLP/O90DM9RwiHAU4v4FvHw1bhkHZIEDZQ0MVK43Lv/a/7P7378eL0/d6vP17uvz344fzjzz//9Ms77UA7/6+DuO6+45lXozXPKgdvP559PHh7dPHu7HKv/PHyp8rxx8rJL7XT9+rp28RBji0wJ7UpsVIFJgf2czfsOa8/95znuee5Hy4vz5ja4HYy+vCfz3PCCd/69A1SVPQucrhDotgVNfT96eW414Tkc1hiQneQ2cV+QMJmFNqF+vPcz4UDH984xC9cgnkDGw6OXNOzqNtpQNnIvSIWl+YSk+nWQFeE9AvYodfgaT9i/7aB9kyT9MO0HhTHZpcw9wl9Zvi+F4QFOGNeNeU3qO+ToTfTLj3PHX3uU3CbBrrA4RskK+gddpkNZCTLDf6/6OkFOM+B511R6OdZt396u8+8p/DhbO/gcL9pYpNoZbnaHvapHUSE8OZlRN4gmM3vIrcAVqwitdyoJFbcQaf4c2GvQ5plWfwba+2Hs4uji4vjw6ZWNbRqrWaVsW7UqxWjLss1WdEMuyJXrIpS3UF9HHabpbH6PeNDH5uFkw/fH79vWgS8mFhDunUj1u8C9BsmWi3ttzKq27hSsVCm2fu906NmfGJ6Kaa/VpEVXdYnehgXO9y73NvfuzhaRskTDEN9Ch7CnH+m/14Q/5r4DeSpwQ0NzW7hzLsh/r7H/OW6POy3iacP3Nbws4KpLOtsOuV2Xxx+OLj85ewIsQmBzj7unxwfoHyhVPqpfFAqHV4eop9/uDw9QTDnSqWj9/kcgn/5ODTd3NwUb8pFz++ULs/FRFUUVik5LlqhlW/ldrlwmMJu0Myoq+i6LmrkWaGGg91OM2/7+XR+SnyCSkwSwVYr92y3R0LMg1gBfJheN/Nm3EcW/qGieNfMT5vseVRKxbi4R5p5HIVdzx+qena6j773vajPy44WviK3N7C2BUPF49CMLIJEcH6DhqfZZIMWCUyf9lkgGRKzH9d5RTo+viY7UBWdDkS/5aIzNPI9wwuH9cGOE7f5olA4PkIkRAHIIC6ENSbJ9HowD6lBHRr+X6GQZVTp58LHPfApUdAhUipdOj5qHlkdIo316ZqSm77nh0N63FALZrtFrqlJCvzNG0RdGlLsFAITO6SpvEE9/Jn2ol5yAnzm2bNnIDqkoUNaBzjEjtcBSxAXObTjZphktyTK5p6x/3Yd6l5BwuI0pSC8dUjQJSSUEM8NJO4TZhBIqOsTuykVS6JMCUzSgxVH/CpCie+UmiprulrRZEnYchm5NpikgG9I4PXI6qV7feIWAli2Vi/a54lcklKsUH6/Z1TaPcuGhhy2qo7qXquvRHYHZgLxKWn3u5D9rKWJ5GAtsn3qhusQHOLOWoxxTdnMdEkUkGzxEI8QXGiziMsSQJHoIYhBw01f/PDh/PLg4yU6PvjwPmmN9jCkmaWAhqRkYwgpME/hhYchEU2FmiLuf8LXWJzNo8A3m3nqmk4EcXfoSsnycacN+bLv9YufgnxrtyQuLCGyi13LISsV2ff60f3CpDFhzCYIPaM22npheWbUA1MXAS6OHMIOg/3bAwcHwXuI29t/oVIJ0s/IR5SwyDujPGoiVsqOXJ4Gb7ler22yi9t/sfPPrjEICkkvaLrkBu35Pr7d2t5JL5leBAuDLE7Yno+2eAU4g+hudtOXuMMa3pL+Q9ouOjyNgcKvX2+jvxDX5Rnv6ByVmV5bdLtoDjrTRIMOINEDEMf0/42r+vr179Dj+WXvCI1YpvRFHIlfPgmBGoVleOf5efYD1n/rCWMijKIQ1uSAwjLNh8SRCMvmwPFhtQvQx/OTgK2Dg5HODYYC8aKkDYW2LFg1xYAw84L6B5AAQUI73BeWE2EzJPz8d1PON0bPQw7Ftaf21rDQJsp/vHwLmVXsBHFvhUIAtCyN8FwQIhTjIr4QJyCjxSED6JPhIsJEA6/PLRieMue898krWfQTDUsQgnpwMYxcCATxryIXALFq5gQTlv3kQYZt0w5YlnWmz2z0wT0BngXa9IGq4CRAMqQ2DSTZfsH2JXaKBofEiCBHtzGYgZ2BkHnmYOq+CxhcJlUBOq5Y2Gug37itWKoFgiDg7uOAcFFcPEdSqVgs8f8lZrjf2UWL9FldCaiah+nSKQkCJvAi9Hwi8TKgnGNAO410Uo8UEmN6Q13LuymCiEPoM+vtFz5G8YliL64CV9jEH5Gw9VfkO6BdCX/CYrui51kRJHvs/StQnnSaSX3pDbIAqMzwGPLEBrfOF+4NrEugLbQ68If7x4hH1Hu9IHmB4MoAI5G7CrFgl1TyvAq3mEt5MOTtoOvdtAPPhES57ZIQeOMKQYycO/4LHbM0DGgPIkw7INg3u2PazSU0u+OMHNo+g50gHJcKhhXolts1POsWeS7b7mF0wF3KIjaOnPAixGEUNCWGXnwr5e7vAPUjICJT2skjRKEGGJWZJz/H7Eyjog2tvPcg0yBx5N6iFndpZuzIabqR4+zwOGzigEdZl5cOEF8VKOtSLomjPSEpe0nYvz22tiTiSK+p9Vo66FLHkraFaCEyyaJzPIC+SMVtzydZCBEieQBOqqeBPnICLiC+MGW5ihwhY2T9RWwBhvrxCovEEiviMzQGV36jvzOBe2HoUyMKQRCF/rHgL3luxyFh0KaBYbXjMXsdmzlVDTQTUsTi/8zwCb4Sx1/SaI/SVYFVcuh99hCt8kZ5a8nK9MKh22ju6kPqJgKS+lxnhxYhqEOt97CW7cSLdrxiRcxzvgw5G5+3N10cbv0ReRRg1xJZhdi74SeFUcDXCJKOfj47aZ98OJBQg9thqq4WvW6Tz32nDbFeQq+Z3CJfAIuxS4GiksE2Y6Wd+SRNleLCQi3t8Hh7j5jYeMOShrIqJA15Qpu6kE7Q68WkDvd0quREbm7UnwbGXY1hhUmWtOtco7OIWUc6vzKTDgZrzKhxmI4Nyjy5idhrMfRO2JbkARh9ONPnkycz+O7kkpw9ctKkO+UGxOPElNDl0Dh0ZQQvXjsrfPE5LXL6WSGsLb1mfRqKXQkNQM35RmKWkwxauN9HRjLksT4MNHkxUJ0xvYRevUL3FwNynqdYEHU6cxQzcHAFGfjAWnOYaygepLWmmeyeYSv6BOxmksTHpTfMvtL2/RM4WW0Gv788i8PSl1GP/8JC+0qSoxBDvwMwPUt/F0kI5xC6Mnk9GpgT+SAYlCVeEg5ZZWbWpvRn4Zjddmwosiyz0lBmqCjfjAeMIX6+lZ7rwYnBW4m9bbOUkPggoFtuJRvfd3+zLW+LIAdn7fJCSdHciOB21/ODtu15IWszHtslDMG2ptrJnsuQOUBm0ixlN/TaMbLEHePuDimqmk+UHFFVcmHidTDgECONAOq7ndYegtQipGJPnoQhgfeA+tdeFDD+vyZ/oga0LwrndtmQEyQ21SUw/0vQLvThx4pbF43weOS5YBPqg9eB+oS4EmrtYgHG+eLYjWP+Lj8qA47DOBGGVoI+dlvnJPQg6yE+whG7Rdlj2xR3X00YHYSviYmAx6Envkvv/vYHeXSxWIQ+MAG7JQw/ocVQi+nNfZgkhmJbgzbxeyy14k7wr9RkbH8wtb4EsiFWtEEiKCAlapNeP7yVhq0ejwj3kRsfCJj75HQjct+Jr8dCTez7MBgBcdrJ3fV0HkjC15IRiM9ywY3Kyx0ptbc0z216NpVor4Mwu8cBU4RaFrvpkvip2PsEo19T0Ecp9t2OVBoYdEIBpVxUKmq9WhOvTJ1XrhH0dx5aPDZIHLln1l607xWlrqAC+hH7FO8aPiq1PmH3mjmYKivlkX4mQyOeyODj0h6M04pVU7hq6qhq9t3Xa/8+3dZiJZWrUh5V5RT7weOqUeZqVEbV2AN7OI+rR4XrUR3VA+b6pBZLzkx1npnpwgIzMStFfGChLicORLhjyxgTOCqB3c9gztxWtM+KVuxQm21DotwUXUd3FLim4Kt3/x2y5YxpkCuiXYM7727JaPHFMzfsOOzui1heLRwSMdsbCImDEQ/PiRpn2I8gxI+VlJWSrJZYMZQTFh8J7bwm6/Hh3VdY0agT5/RiMWerQhKmgVCwa+05Tn4wPINVuZFe5lnUlrK9I/bIhrez2thxtuILHHiuPRrAcI5cjUczdk04bPedKGA/42MqmmTVxGgYng+rW1OSJb4MictwVYpHXKxxI32CNyY5h/XKCS+4Evk0rk6WNj3Hwf2ATLXBUIGt7cl+9DzqBvxlvCdJxal9SQpM9CZJGSDAhm32GB13QBEaXzE/xC52bgMavBL5XDOPQv4c21CVJLPIMyUFMEApdse+Ke198iCh98GvkH/3ldkJh+CQbgBJUOizfIgv8q35ynHNOUQBY4UknoCztgrZP3DXdpyvTPrUb/dejZHz9yZMtUplJ570aQ4dw0WaOxKHlzvje0qpaYT0gthpyieKDzlH0rwF0EPZ3AryyHMPgKeukrnD7giw/UcuX3rD7yFs76DJebIliSJTp8qgAIp3uvhG/E6e31dECHTLmEPC6yC7hOEHJ+Oe4no9OGyyy688/ryLS5ymko+fGaG9I0g8h8xy3OskvpG37r4CSTj5xFnzch51+Yxq5svs+ZY0iuZHoyhzvDSK5iFwh+BokGebTvwIIEG01/dp7xXBJrjVjpS6472lSlPHpcC8H+cRu6HcYaPDLoi34m6vKCa6Ku46w4k273J+NAoEbB+cpd5gpNGgEifUzXhwUGJ/MfPavAtS64zdN4xc1L376tOQ4QR1r/mzlhkxJ6krHoCCTB+VOG3d/S+4Azr6+ONRXAll1YKEmnosDrWoi+Kl5w3i+dLWyAqyvVuiWQFS+rWs1+up9c3wc/saBhrOFuv1girLlVe419/x7bB9jZ223QubLKC8LO95Nn2pvoUTcHxFruG1F36G10+MULCTVCt2iOuTJhbDmp7FokFmqpeqHLnw0n15UH65pwuTwfvEaGmdT6IOK1bXeVfTS0CqEWmyfqen2JratIVMbgcQySyRFiCM9JpKpQ4XCvCjVJVXrJ/gIAM1I0ZhTTYMQ+c41TXZKHHxNRgpKTXuyK90svJVOA5MAtczIhK/YTEekExxUqRQgFJ+h7oFwwtDr9fQ+p934vyjwXZY2F2akTgDngpj610RSJsqpG5VNNsoV227UlVsUy3X7bJdxzWd1GzdkHgaEtIe8++aKtflek3TpFm8ths5fAHLuAMRz5LY07JuUYinJJ6xJ0+GhGRWz9juGyzR/0o8OH/GS7DIbIrIPLghIIkHLCEuTw2uorZYfh0aj9KobkyP6ZqlO2tZuh1f7B9ma8au3KcXuz6qVU484vBM5LWRIxLcAXRLvncz2Bsa+BrfAR2xb5ycJ37EF+8daTRK2J5rFeI5LLV+i49+j708TqjihDP0k1phGhIZBMTZu0873TC1ndEpdHxyO9YceOEfbbMLM05qXVKWXTTSeSWQYrR8bPy54+8gSx5V9rZveeZy2sJawzP6eDc1ydXH1R9e3e6+TlMIFgUSsUVhCY32hAw0qUPqoeO7UWItghBCGNVAVIUFojW+KOHWGyRkT9P+FkLQcqq77t1Xbk4xwPwZi2kWFdCZrYnb5w82LKWLcHn27A8TthVsZ1gUFhOgYVhIpinCHq6OltTkRMiYnBJihDANhloX9GdAesAYFE0d8kyYRWOzjCUbySRLOTfdV0T3iJ+2tTPWAt9JihvgScxkFjNorig8oSB+DZuex6FcEvv4+jsc/YYi+Fjs42voA0Lf9IiDfhud6b9DereqeSfaHjUDz3AyzqfmKcYuMcO7RrrXQAPX+gYdKtkjH9krn9hoERQ7stRm7ICMZg6TBTKX7FHgHRUxci3dfB/KMR+Tvatzsnd1zexdnc3e1VnsXX189q5+e+xdfRh7V6ex97EbEt8l4RuETfPuqwMBGbNYyR8rC/nnCj2/xxOJIL7NeR4FASVzovm7yKGQArz/cHB0eXn8lNE8seRL9QDgGKwpUNqJKZ1bFS5Ar+B1zLL8PLw4GF6EfddM8VUloXhtKsXHgzYB8mJMV0Hw1fUSvK4aFdNWaoZdtTViqnZF0c1KHcsavGoVcz0EX12O4KvrI/isEL0AwWdrtgKCz9JrbQRffYIEv3QU3wD+AoCvKOWKCpnz+Nq2IfwMwq+yhyRgkdkQ/qMD2ZDp5yT8ydj4UMJ/cERaYgNg9rxcaAcgMd9mB2Bhh1v1DkB1+R2A6j9zB0CbcwdAW/MOgDZ7B0CbtQOgPf4OgPbt7QBoD9sB0KbtAJxgFLCnn2HIeuxhpznB/pSaXeKgi6Pz86OLp8z1J4zJuYXgd2yjNcO5lsJ5rTwNzoXhJ9hcDMsq2FxbL5trhkwMrWJYcrlel2uaUVNsUzNwBSDcqOv19bC5thyba+tj86zguQCbZ2u2AjbP0mttbK49QTafFl83yL3QPfWqDpnq2KKzAe4M4NZ4El6bqsgGuNfGP0OmnxO4JwPeQ4F7PMwsdSP9/sm2EEUnNtlQ9MJetGqK1panaO2fSdG1OSm6tmaKrs2m6Nosiq49PkXXvj2Krj2MomvTKPoI0DPC8eeTAgQB1yImhOOA2YEAbbI0PJiTrd8R7BZOMUQL9P3Ho48nJ0eXl0/6sfaB9dI75D2BybEV2R31ITuumb9r5YS/69Vp/D0YmMn74+nwrYLDa+vl8Gq5XKsZsmobhk1MC+MyuFRdtauKiqv1irweDq8tx+G19XF4VvhdgMOzNVsBh2fptTYOrz1BDl8sQm/ofJEb4mUIlOzGW/a6tcH0DEyvsY/Fg9k2mP7ogDVk+jkxfTIePhTT54tCy9wEn3cuLkTxick2FL+wk62a4mvLU3ztn0nx9Tkpvr5miq/Ppvj6LIqvPz7F1789iq8/jOLrU++FS12v1yPIg5UWcnYKKabL/461SVwLVvMAWRGglNuZ9y75B4fyPwl0fvTLU+b3k5dqjVsOmHlgO3jDb5rH9oMjsCB7H7E76syKa+b4ejXheL02jeNhZCYAPh61VdB7fb30bqhV3SprhmIqRLNVUqmoBq7ZdkWtahVTXtMT7vXl6L2+PnrPCrsL0Hu2Ziug9yy91kbv9SdI74tH5g3BL0Lwsl4tQ/o7sl5tuD2D2/nfb4P1ZMPtj45UQ6afk9snI+GDb6/PHX+WYff7Z+FCxJ4Ya0PsC7vXqom9vjyx1/+ZxK7PSez6moldn03s+ixi1x+f2PVvj9j1hxG7Po3YDyHq9qlPw4g47EsjIQr3g1uz6zleh30Rypycfuj1qMvKo62g+GkbXeydHL9/ysR+yBk8sRwcsw+uCzIfsd+aCV2vxYSuKlM/hs7HYoLR0xGDyi/VOozaS1VfBbHra77fTipYBx+SNbVqm7qFsazLRrWmYFupaPqaiF1fjtj19RF7VthdgNizNVsBsWfptTZi158gsc8bmTecvuBHzyuQ9E5ZrzbEnkHseg3yelhfNsT+6Eg1ZPo5iX0yEj6U2GfFnyU/aD7XLFyI2BNjbYh9YfdaNbHryxO7/o8k9qo8H7FX5fUSe1WeSexQ5H5i5wUel9ir8jdH7FX5QcQOgzP98+a2R8U3IUQuQWbk8K9P6kFMp7D8BpBezEnt31Ps2uybtjx0zmYMfcrEzj+HDpZjmI75n4yL+N312H7skfnEgoK718zuEHdTdten3l3nwzIB74OBWwGxV+X1Enutpte1St3QbFU1cBmXYfU39ErZLGPdhDNrIXY2f5Yg9qHqqyb2zLA7P7FP0Wx5Ys/Ua13Eznrx5Ih9kci8ofYFqZ09k5uxXm2IPUMHReF5vb4h9sdGqmHTz0fsGZFwiY+wz4w/S1L7zFm4CLGnxtoQ+8LutWJiH8oqHkrsQyL+UcSuzEnsypqJXZlN7MosYlcen9iVb4/YlYcRuzL1+9nY9xcS1+Q5xDmBbAIddLE/J6S/x2EXgjdB7/f2j84vnzKin/mCmrmxkr/pDgYTZ+GYmW3dVK4nVK5OpXIxEhNUnozUKphcWS+T20qlUqlVsF2TNYKNKrzYVdOs1ohuVetVbT1MrizH5Mr6mDwrsC7A5NmarYDJs/RaG5MrT5DJZ8TeDYYviOEQGVoTK9IGwrMgXGepurqB8H8DJQ1MPyeET4a+B381W3bAWZK7Z0y7xag7ts6Guhf2p1VTt7I8dSv/TOpW56Rudc3Urc6mbnUWdauPT93qt0fd6sOoW51G3T94PW4KdPc//A/CIfHHPC7vvv7J8qy52PsMogMknCd754dHT5q8E2Oxh8EZIstwwIwGv7jZ4PelYOc/xa81U/h09hZjMcHeYqRWQd7qesm7bGqqocq6Vq+WKzLRq5qhy3LdlsuWrFVJdT3krS5H3ur6yDsruC5A3tmarYC8s/RaG3mrT5C854q/G/5e6EPmMr8BN7Yqbeg7Q4cNd/8bOGkR4p4MeQ8l7nsDzVKfJp8x3Rai7g1vL+RHqyZtdXnSVuchbRgatnL/EXkhmwXdcmwVJjJkayhQJumB3ajPYtDR4E1qD6iTi7M9kzhOH1sWdTtNabgLAxGjGWFOpITdkc7GjMKPTUNqHXgWKRjY542G3fvLQz9YjZDcVzSEVbbtUAPUhdIXUb/v+eF9FcBGfA0ZVDqBMw4N+Nn7agbEHK14IU4kdVqH4JaeSw0YoZAv6N2JVYBc869fhvhveg6zelOCDBtHZuSigW0H7pYb9UPmcPFovygUkA3Id3j8I+ph8CiGjm3bA3v5qFBopWwhTuVbuVzqD/G5th8EbD8hl8DKRAGHulftIOrz7ZBkXubH5yXDQb66cjbiRyCjlm+dQvbDvybSufvawQ4bdJhyYqqljU5p9YYYAQ0JaxkajpEoLmBQSAW8NtvAyMc6dcOw3yiVxJWwS/6ISLEHGAjNW4Q/M0SKtp9ubJyKSyyveMsvgrLjpzKUfTZN237PAE0TC8XK3NzcFDueB9QATaMQyI2EzbYBacBV6y3/QyTsWSmfmF0Cs5kgfE1M9D2vMNz2syEDsK/HZI3FAz3We9ZgQDtG0SVh2tVisciaCTwn4n+8FNwXWu4Do7Ov7mR/xpTisHv3N7NYsZhP9JSEolILWuPapG7Cw168zKLcqEeWYq9jHsm8kF9OPJG6oe8xM42cEKMpNtxY1MoYBxaXRukm32ePSozGzuQK284RTJqH9Ds6JL4HAM/29J7nEAKaa2V5cr61Z5oRoU6KWROl/5VnmTTELpS4Gbca92pWGIpDu8kmkhcF4P9RPrkkJMXWzQvrDo9fMO6+JsweaIxcw0vR9Hr51gE/gy7u/r72RbOponM2APPULFKzDw7Jj0tFC0bk2A3AWaIQHYAneA7/XByY/wyymQe1YkTFyHRu52rk5JZF0NE2GM5y648Nw/RWY+fPmPHZ8zp7gEsQudgcGfaK78WpoUpcvSHfS44ypgF3cDYP+JxJ/d7AIB9HbTU/uoIPVzU9N2TfCsSDeU7Uv38bnbo0bLOdza3tnWelUhpX/Bcv0s1wxPU0POtWaNwNe3zn4Hnuee7/AU4MDHiMvQAA"} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763514659.22/warc/CC-MAIN-20240715040934-20240715070934-00042.warc.gz", "offset": 116903011, "length": 6579, "data": "H4sIAAAAAAAAAO0823LbtrbP1kz+AeHeMe0TixIly7YkU3tsWWmc+nZsp+1up8OBSEhCTJEsQfrSTr7nNN+Rrzl/cdYCSIrUxXHq9OlsJ5ZJEFg3LKwbQP14cNmvmUb9ReVHuKpeP4SsQyImwsAXLG08ojE0NuqN7Wp9t2q2rus7nXqr02j/nHa4ZE4QudXjow7ZTyK/kyTc7ZgN5g5HTrvaaNX3qtum2a4O63Wz6uzuObt0p95o0b3ei0o/8GPmx9UT5o/jSYeY7b12c9asCKJh6HGHxjzwa5M4DrtkKsYxPLLmSP2RRg73R8EcLaPdtsN2hu2quU3b1e1d4ITuNfeqlNXNJhu12q0W0iJBAGYniSKJPCgC2dtuOnsua1f3dswhMOQ2qpSyBvDXGO0093aHO62dDMjxRfXAdYE4ARzV20ajuW2YOztGIxc0jcYsrr6/PO4Q5Eh0arUhH3o8iCfst4QZUy4EsOuyUUR9hxmjqMbh7t4IJ+G/vFvPokk8CSJbMLbOXavZMM0U9EUUxIETeAowTO7ig9gT0N7MOObhhEXVq4TjPF+fXNn9twfwv1G3L85P/m026y37Cm5bOxkg+uAF1K0e8TETcYeICTU7F6ffvz84f9vf2f3h3c9nlyfvW2ffvzt6f/im0f9p8K51kI499ALnpjzy4N354Merw93TH/797v3p9fnb7Z/OB42fT/vfn1+1Ds7OMpm6MCd8xJmbE7CoHfeTeOq9vp96LyovKm+vry+Qf9DdOjn//kVFafJp4G8Rs0XeJZ7UapLpM/nu9Hpe9WJ2H9cQaJc4ExoJFltJPKruvaj8VO1H9M4DyV3DFIkRXAx8J3C5P+5A38S/Ya6E5jMHaeuQG8bCKvX4LajrDzR66JADx2FhnI+D7tSZMNTBOMKJCgMRV6HFubHqWySMWOFm1aMXlcF9yEH3OuSKxlukbpJ31EcZ1Em93pH/FadXoIH9ILjhwOfFJDx9OEQNrJ5fHPSPDi2HwprBuS/qZZcwBdx6E/Et0titvkv8KkixRUyz08ik2CWn9L56MGZWs65+5rC9vbgaXF0dH1l112kNKXPptuMM95xma7e53RptN9s7bDhsNtpdEtJ4YtXmxk+H5yF1qifn3x2fWS7zWMzcGW3XkwT5rgLfVbO9m/NtlmmbJyoFipSdHZwOrLRhdS+gf7tltnZa7b16e3m3o4Prg8ODq8FziDyhMNWnoCGo/F/U3ysW3bKoQ4KGuOOxM6leBHcsOgxQX26bRb3NNH2mtsNomUWub+/gcqrsvzw671//+2JAcEGQi/eHJ8d9olVrtR+b/Vrt6PqI/PT2+vSEwJqr1QZnWoXAj4ZmCMzb3d2dcdc0gmhcu75UC9U0cVB2bbixq/Uq+xI4LGFfWEvGmu12W43QsFPHo/7Y0kaRlq9PXS5QHSEx6vYqa/tTFlNpDaugw/zW0pyUR/QhMFDdWdqqxa6RWg7Gp1Nmacr6FoZenB6S76IgCWXfcucb9nAHDlIUuqfmnbiMKAO/RYrLbBGhy4QT8RANSQHMYTpmnY0jesu6MJSczkC/kaCXUBQFwyAu0kM9L8X5slo9HhAWEwEwmA9mDSE5wRTWIR9yj8f/W60uE6r+U/X9AeiU6ugxPYeuHw+sgTtm+hxPt5zdhUEUF+i44y6sdpfdcodV5c0W4T6POfWqwqEes8wtMqX3fJpMswbQmbW1NQAd89hjvT6NqReMQRLMJx4f+0tEsl9TfStr+G/f4/4NRD2epYv4wWNiwlisExlg6FInHCF0MonYyNKNmupTA5FMweOoPwb0+Je526jvtBvbO3VdyfI5cEcgkiq9YyKYsm8PPQiZXxXgtr496EhGg7bLRejRh28IP5wOt+2pOwJEHnrVMu27e98E9hhWAos4s8MJREt/C4rs4m+BHXE//jsAx3T8twjjluPK9Fki2HLwYI8IPLDR4mIAqAI9AjaoiPrq7fnldf/9NTnun59l2PiUQphZExDX1kYUTAqsU/iQZkhZU0Wmsvsf6C1VrRoRkWNp3He8BOxu4UnNjejY9m03CkLjg9B6+zX14BkgJ9R3PfZNQYZBmDwOTJ8DhjIhZI2PyMZLN3CSKYjagAxl4DG8FIcPfY8KcQZ2e/MPUqtB+JlEhDO0vF/oTyyCvUaJL8PgDT+Y2g4+3PwD29duKQCK2VRYPrsjB1FEHzY2u/kjJ0jAMdRVwyiIyIYcAC2E7y9HfU3HiHhD/y990/BkGAOdX7/eJH8QScuaZPQJg5GuDb5pODNmLDJjgCgOABzS/4sk9fXrX4Hjp8PuKoowUvqortSfiMWQeirJSOZlO/6C9N8ESpiEkiQGnyw4uGk5JZ7OMJoDxQdvJ8j7yxOBfnA205XZVBDZldnQacMFr6kmBMUL5PchAIKAtsgLxkTUiZls/9eK9k65HWIoST0fbRSBWkR7f/0GIqtUCVJuFUGQFWMYEfgARBEmQXxknmDl7hABhKzYRYlopvWVrzRPS9d88CGoufwDj2tggqbwME58MATpH0MCAFv1xQWmJPshgAh7xMcgWWQmRBmd+yeQz0K2GSVsCxohSYbQpkP0UVQdRTo2cXHEhgnE6CMKYsAWMJkXHuX+O4HJZTYUko4bNHsd8ouUFYZaAAgM7iEVTIKS4GVKqhtGTf7XUXC/4kOXhThWh6xamunaKRMCAV7FQcR02QeI84aAp5Mv6lInNad33HeDOwNAHAHPyO1HOUdpgzFNh8ATXPglCBt/JJEH1NXoB6pKHtPATSDYw/t1IJ6NrWy8vkVcSKic+BjixI6UzkepDcgSUAtYZ/rw+BxJi/qoFmQfYFwxwcjgfguwIJcc8lMJ7qFKBTDltpgEd7YIHAiUbZ/FkG/cELCRT7b/isZlFAo+BQtjC0YjZzJH3ZOALmccMwc7wmRHxPNQQbAqdavsDwP3gQQ+lnswO5Aq5bIRTbz4KqZxIiwdUy9ZSvn8pyBhAhmRo3c1QjiMAKGieLQnrM7cKo4Ay1kAkQZLLfcGd6VKo7ATz/ITz+tKO+xQIa2sL3sLIr0CR5YqmR2dKkjLXcLhw7G7oTNPf83d13p/wj1X31SgFcgsiq5IA/oyB7f5NMgKiAIpDXA2PDf0iSckgPTBCneVeApGyf8SdMAwPvWwRLlYZZ8BGTz5hf+KAA/iOOLDJAZAHPhD468H/thjsbC5GLp2OmevUzHnpAFlCopy/mvDiNEbdf0xt/Yk9wo4yOOPyUNhlUgltswzvfT4Jnny8AK5GYBsvKTZ4wYYdRh1Br6smzrt1GMlqDkfC8om1+3dhMYbvyUBh2TXVVGFqt3IRiUU0DVG9MFPFyf2yXlfJx0ph5W0uvzWZvehZ4Ot18lrhGtIB2ikKgWE6kMsxurdp0FaCcUHR613pb19BEwqvCKkQlRF9IIm2NyHcILffh3UIqcrIWdwK2V9mgn32whWieSZcn3S7HyNWEvMfzORziZrTqipmU4FippsEfw04uAES5J9EHox0peLZ6nx7VaymD3x8qA7zxuItBMrTJfHU9O1xHjJ0cvMl1zTKqb/kgmz9dfIU8F2ZdkAjHzaTHxJSWYYHteRUoQ8x8OMkpcz0jGn18n6Onm8G2TOT+kmkvH4Cd2GVNxABD6T1hPEVbAH+ahVIntk2oyIgdwclum4voXy1TcfX8CZt5n9/biWmqWPZY3/iKb9mwRHMQW+BYgew9+vCQifAPSbwZty4SzEgyBQDLx0GuNgFKul/149xq3Ljlmv17E39Cl0lcV4SGNYpPXytik0zG51vLUxJGQRAJg0e1nh+/OfWPJ2GfHosiov9FToSoDtSRAJexQEMeJM5/YZgsDSlJ3VXAriAJgZWo4benaasqSMSXWHELWhZUSWSNV9WHhjCukQZhoCxvvj3gGB0CLmqibP4pjBPaT6t0EiMP+/Zb+TDuBXnSv7OOWMqKK6DuJ/BdTFEfy6KXaFRNqjwAeZ8Ai0DshnzNdJb5+qxFgz5jaf5Z1WhgHXcRoIAxYRUr93yeIAoh4WEZrgFuUUyxSfPzkwO4TeModAPg6cRD7//Gc0i6MNwwAeEMB+jcJv7GKqhXRLHWaZoLA0OGLRFEMrqQT/yEWG9cFc+jrABlthA0QgQM/IZtMwftCLwP55DDn4mA5pxGMD94G2bolpNI2G0SSNurmD5x/qu6S+22ntdVq75EMI2KnvMzK4D8k/JVKcKtD6Ys3OEQLUzAA1yeJ5W00M2MC0AfJ3bOlWPlbKHaPgbrFbFVoXuzrM85b0xWawTiF1cY+vGvHxJO6QVniPEPZVQRbJzlRVgmSwIj0hFwcuNzWdR58/YbNcbjp0Y0mUzxN0KizpAgy7sLwRWrGLYCxTovxersdUcdNH8sYudYAeUQYGVjOPH+bwyC5utriLWNPupZkoUVF+0ss9TnHRKjgeu2Veo9BlTYkL7g+keNbWLg++G5xfk2uDZJ1SWRXHPPlnjoSSlsg1CTOMgsBTBTB7RSSzy8wALz55CpoMuo1bgoFg7lPRzP1Uyh2KN2rFp1eRUoh05SP8Utfc3xQ0Dr2c3GxcK2hviZ9iRzvmcQTzfJQGFGCROB4qwj1MIa0Vh7YQS9EsJkvUvmRmvJgCNGeSEVHCDTE19d0Dz9Ny06rP/Egnfyz9/oa52VVVnWIBxqaet5E+kCH6bcAFuLTSU0DLp+PMzsGlHXqJwN+ZH1N7IwolDjPGfKSTYRCBPbb0ui45Uo/hqZ6aYsV2iSe4cdilZP1KEqH11v2hCLvLejuB59FQsJUyKHTY2FzkYxpwX8iPeU6ygSt5yTo8l5u/GiUI5jFnodSltLh8K1MbVAF7jHF1EMVya6RYMcO8Keud1xKy3na6nb7hMk82YFmhsFuRplsR5luP11hx8LpIhlYGej2ivyuK9NcbKINglGPZlGF+gmfWwAy7Om4RpI9Ih+j65msdYIVhZAM1hfGStoWh0JqPwhjHZj7wlUTCggDkVfNodv7tVWOHu9CCZ+BmMfsaKYvQyCQHfG/Vt3R9y9xKi6YzicGD+mYh7F8Q8cKIjewEYlG8K9O5UQTZjZ3W9iAgwhMQuBUEIUSEJ2esMsl4gSq2MaNJTh0mbo8VqlagWZ4hp6qboSDqR6bL+VYb35f7v+leWjFRXssL/eyWehvYDZMvpHoOIsnTqNlFLtcb7ilNsSXtG/PKKh8hG9ZXMj0THJYPcjCbM/LztkLVDpRlGtwyWZEtDFpICBcXcsGWQO55Jdd8EElbF/h9SGZvLF0qEfhTFsHELq5asN+EerGlXUe85Iw0Ik+tLHuQZloOrA8IfvHgIGQcLMo8QdlUSrs4fLDp78pQUjwlY+nDII6DqU4m0gZaenPBhEoycRu9bEBBU6bqSI8+dWxQA1hnAFXOrLKGxdxhGkTMVq5S5CliFrPKvJtARD8JAB+eccTkh/thkpncCXddTE4UPuX2fvlVJxItNNjbeq33yAgpF7tuY8ds0OMjgnCu++C/Hx8w4sxz1ZgCZbTT6PwhOuZuZ/23JIi7gkUcJP87c1PnrZq7otMw0y60Y+KQnfT2anBw2X+bdjvrfswHmGYOU0YBSNQMWr38EMKJEMJpSaRIe33Ua4+KGQ1wzogpO9dw1p/qPFcHBHSEFj+yMWZiMIIVwoKi5hQ4s7GC5FpgYRg8zqU3xa1sAGfBr61cmZ4uGP1AYcEqQY5I7y1rLeq1LK6hgDA/Kvjp1XtI+AOBamqFxGLo9sujT1MD+6u1vbPX6FYWLEylGPJqzMNuF9Jo5QmMgl1VpizLlYoCz5C7LPQ4VdYjs0xpgIobxbgtheD1LbmzDHZqMRbd0GWPldFo/pykux9yc7arybMm6GKWRakqhtKMGthFsE5y8v1gCpcWPl4P5BlIn3mWqaXnCPl0cK8VRHI8Heem0lU5q5aZMq2uZQZOa+KRx9w+arl9RAtnU9vcuTd30EJq0hzrWYKgDoUzAuso4tN1Rh3ICbq5rn2hW23lnFSxwEU1gmeMxjgz+EDdqmKC6qY4VQeRoMFGjrXyohK4M4rFGBBROcxNSyyWmhmSCV+V1mzJgd77nrPoho0pjVw8CArLIwRHGIgA/jACSaZAPwX2w2NLlnQGTAVoEO2SGjn5/AmcfP/t4Or6/IdlY35utvf2chE68b19C5MFrcbeXrVRr2+v02nYjUYxuhd7NI0tfM3jVfMgGPFXjTfQANc37BY+p/E9fA6D4CYbA6GCHzGr1DRUqAqsvmrUWQwfHoWPAsPYXmQ5B/ENIKBdFVZzbxsH5M0uxN6W2W621pFhmO38SQih+DsDOv8QcX/WikVtCwwBF3kbTbAeZ6UiLzTLEp918qrffHXQDvz5AapeARiujflHV/23J+eDnweXC9AOA0ANY1ym5+pW+pMv+GFEsgUwy+LnLJqMuuYNmqMaVbigTWk05n5VRSydnfC+WwwgcPO/ZKjQZ5E4uGEQjoy2KUyHM2q4zbbj7Ow2GvVhy3T32numydo7VCco/pij2zN3G2a90Ww12lg3XVlLlXWtxJMh0pKNbbXUUkVftvGtzt6t4XnGAoxlo5fsIc3c5T+y9aNdyB5o1x1l12e7zLo6tQ9mfZVxVoOVH/R4OkNlypCMlXTlmzXLKDu+OjxaThc+eYQqfFymqaIyizVVHko8VSeaVa70KLib7TYUSkW4p1aU7VwIKr1+N5twnOMMYJxbSCygp1GzrKfmghiOq+OIPZTrPjqo02+2M4Flo/eusdoki/RqfYD6oAqV+qeS/OvmWEKVFfMy9Q8hpE/PIx88ERaAszRMRXYdMs9P0fd9/rSKIFVNE8+iSFVasViwINNlgeSSd+fMdksnvTkfRXtbRIEmXfJUSFiBIL285qugXEfUTZyvhGS2dwCSNKwo7tz2zoNcJVnmms+S6sDlkuJFRV3FgFRaAVF0xkNrFziQXgnm5p1B0GflGzpLSH4AW/08TfD9z5+kdqoFJM84rlJQ9K+rKPGlX34WLcdTfLMFdyGXKCY4fBKuQi4gJXwW6jeQmNF4CVqzTZzpKrT4HlXyTKZPFIxFpcH92M//Q0ETvjgWNB5cO/UKYMhlIgRbRbisVD6L7D6NP38aA2ImVmnLSq2X2DON326ixr89Pjm/Ooc/A6ntMuRZRXy6aRM/T9ckRQRSHQcyGIgvv5aJjIqMD3OvDYxs00NkQE1Zb/DT8dX14Oz6+ODk+Op0QKrk9PxocHmW3hSc1BecUMgiEcjqw7OYPgumJIyCMFrJbtFvXq0HydTrQlJcICvdU6pkMYKMUItRwizOWVKm6n6VZya/lD3ir5AV/X/wT2pyDPWnmtaB/oILySa54Ee2iMyRliCQ9lVddom0eka6DL9gfVIVypD9JbM1B0PZrmWx5H8M138M17MMV3ZqaMGgyTpkZbb1jfZM7nLb7J5NwYLhC+gQ481uyhva6ekLsFvpqRFLn+V6RRDldLii8uHJrGfopae2VHXQGYLOBi6rDmkkkcaTx/sHuIXeh8/HusaQi9geH6KZ1XtXSYiR12MDstdfZoNO8H0bLmTrYyOF+iqHAjbVkI3pHaWbMvietEx7JguaxG6ZrxePYOWi2d7d2c6UoPxQykGqdflpmfUTfhux+S5LmGXkgoH8k3siwOaf0shfGLXA6CkdU8H9peQJHicSB6gUuLfEl/qTWVx1Fu0o363Kj6eRmeZWiqqbu+Al5xtKVYfF50vz/XKZvQSh9Cg/CpZ+Fsv/xUN5Q5mmTCR3DsPtpC+/XYLHywqn/mMmYhs3UDbwY7Oy9kelonYo8d7A9MOQuy2kR0zcq5S7oWAho3hDO0sgfogCmefg4T+QZkg+f/LY7edP5KWWv6mRwhlBpi7SHeRSUaWS7nAvQ7v/RLRDKr4aZd6GGwvydQh5WC9/l6pw+EhJG9/eKewx4uXq7cXFiAfMLKx5/CIAYZnrXloFsejcPiMJfJEMpzzGgzyKvvIsyZ3Ihc1UWQGVNj/bTI1xBzBKdy4X+uPO9Hx3uVsdJPhKmRStWTOXDfVBreaHYttyRFizne+NbXq+eZcFvvu1VI3z+3wNyas1tfdUejo7iKWOYXKfHB3/QOaP1+KRyLy8m5+4reRuKW2zI3lEEl/8UvXihQ74GqctklAdIlx9LnUUyNxdlqjlFcDY1XqnGIfDBBMPAiYVpUGAoBxwjnQF1js2xFfIETMgTjUz7aC+tMNGxdRSmtKvLPnCtznlG1SLX9fRW3KQeYHYtVXUhtMhrpdUQoXvTxkHwdhD1CSWXz5l2UPI+W96b9CVk8Rns01QdTj3OzmgiHutIACPMx+RZeeoy9wjQsHHQ8Nncc6qYRiIRgReIq0gOBfArF5bpgSsi8tpeqQbumoZnboiVO8BNklNriayCJwqJKmUNbKWah1qpDqYW9hokMextd78+Ww5Z+kJVAiElh8onysza6GpzXud4slutS+gTZmfHLEoSICVWOu9qBCy7/GlJ6y13oHjJIx7eb17ofc/NCx74pHpYfEgvNTqF/JbdxBvthsIlgX0P9GyRwpSKl1NSbc4fwtfRobmAZAxCC+E4QRTrdeXLeTq85+3kUKbE/pEBPhapsGdEL/ZDK9rhgszcuwLUJYkJpDaTAKPp6f7ZQr4V7AMEyNxvIcnITl5CPx5HLivIKU/Nw2rsabKv2TFL1/Xyye4ht+Phl9oVtCK71RTYZAkb95aL18GUsHlt2asFRfCkAJ8mtjz7yAUh+YHq+UyUuMfD3fwG3ts3KGGGGCtVsvtSvTyZe7miaQTX+9VFONxe7zCb5r6P7BygIeWUAAA"} +{"filename": "crawl-data/CC-MAIN-2024-30/segments/1720763517701.96/warc/CC-MAIN-20240721121510-20240721151510-00045.warc.gz", "offset": 104033233, "length": 5171, "data": "H4sIAAAAAAAAAO076VLbyNa/cVXeoeNbieAL8g4YgzxljJOQMstgJ5PM1JSqLbXtDtpGahmYqbzP5DnyNPctvnO6JVk2NiED8+8mgOVW99n67C390rnslqulyrPCL3ClD28D1iIhiwLfi1gyeEwFDNYqtYZe2dNr1WG10ao2W9WdX5MJl8zyQ1s/OW6Rwzj0WnHM7VZjb8eu71qwZlzf1RvV5r7etBs1fbdC92il2aw1LKv9rND1PcE8ofeZNxHTFqnu1Pbr82FFEA0Ch1tUcN8rT4UIDogbTQTcMpZI/YWGFvfG/jItzSZjgFCnO409vbE/svRmrVnVm7vVxs5eY2e0V0daJAjAbMVhKJH7eSDWeAwc7TV1q2aN9IZVpfqIVmy9Ph7tjVjdblZsOwVycqF3bBuIi4Cjyn6pVm+Uqru7pVomaBpOmNDfX560CHIUtcrlER853BdT9kfMSi6PImDXZuOQehYrjcMyh283pWAa/OTMHMPzBbeYafMocOjtS24b9Vq1kYC/CH3hW74DwGt3hoQTwZbXU355MGWhPog57vKwPzC7bzvwU6uYF+f9T9V6ZcccwNed3RQQvXV8auvHfMIi0SLRlFZbnX79488f3/R3Prz78OHX44/9+qf33W7/faP+c+/TcedTsvbI8a2rxZXvTt8Mh3vvfv7wqVG7PP/l7c9vB+8uhjvnw7cfOh/OMx07sWFH+JgzOyPgrm7cTIXrvLpxnWeFZ4W3w+EF8FkFza2QZwVbavEg9rZJrUrexY7UaJLqMnlzOnxWsBK1ExK0YDeijCAPiDWlYcSEEYux3nxWmNHwtkU6lsUCofc8y7e5N4Hl1JoyHYGEKOjAj4QOI9aVUdkmQchyX9bdelZgNwEHzQFaqdgmFaCVeshDhVQqLfmjaAVyAJV/xYHSi2lwenuE+qOfX3S6x0cGkMJ2ce/yWnVAEuDGcBoD7LoOctBBDjukut+qp3I4IKf0Ru9MmFGvqH9L2N5eDHqDwcmxYVcrlfqotk9Zza7aozpr7O3vNveoZdf2G7Vm/YAEVEyN8tJ6d3QeUEvvn785OTNs5jDB7GXaqjrwrVf39zK+q4u0LROVAEXKzjqnPSMZWD8L6N+t1fb26/X6HQ6TacedYeeoM+g9hkiHwla7oCGovN/VwIiFMxa2iF+LrrmwpvqFf83CIx/1ZQY2+1HvhvTaAYtNdZUl6tcio3CFP23sVCtoDoXD58fn3eGnix5BlSYX74/6J11S1MvlX+rdcvl4eEw+vh2e9gnYTLncOysWCPwroncC53R9fV26rpf8cFIeXipDq1ZxUXpdsoVdbBcOJXAwQS8yVqyt7u/vqxVFnNRyqDcxiuOwmFmYJk1MQ0iM2u3CxqHLBJVOUgcd5jOjmLdSWKi+GcV15lok5QyMR11mFGkspn6YW3pxekTehH4cyLmLk6/Y7TWEtyg3PXHOxGZEuedtkjezuwhtFlkhD9BJ5cAcJWtesklIZ+wAlpLTOejXEvQKikJ/5Is8PdRxEpzPdf2kR5ggEcBg3piFCMnyXbBDPuIOF//V9VVC1T7q7zsQ/NREh2kZdO2kZ/TsCdOWeJpxdh34ocjRcc1tsHabzSAy6fLLNuEeF5w6emRRhxnVbeLSG+7GbjoAOrOxsQGgBRcOa3epoI4/AUkwjzh84q0QyWFZzS1s4P9Dh3tXkLM4hhaJW4dFU8aERmR6oEmdsKJII9OQjQ2tVFZzyiASFyKG+ijBjJ+qe7XK7n6tsVvRlCwfA3cMItHpNYt8lz09dD9gnh5RL3p60KHM5dK04gnhB+6oYbr2GBA5vuexRdr3mk8CewKWwELOzGAK2c6/giK9+Fdgh9wT/wZgQSf/ijBmHC3TY3HEVoMHf0TghokeFxM4lagR8EF51IO355fD7vshOemen6XYuEshTSxHkJeWxxRcCtgp/JFuSHlTRaby+5/pjKrRIolCyyhyz3Ji8Lu5O2U7pBPTM+3QD0qfo2L7sKxuPALklHq2w54UZOAH8f3AtCVgKBNCNviYbD63fSt2QdQlqC96DsPL6Oi2C1lIdAZ+e+svUi5D+hmHhDP0vN+ZTwyCs8axZ+HObXq+a1p4c+svHN+ATJjADrmR4bFr0glDeru5dZDdsvwYAkNFDYz9kGzKBTBC+OFq1EM6QcSb2v9pWyVHpjEw+dWrLfIXkbRsSEYfsBjp2uRbJWvOjEHmDBDFAYBD+n+TpL569Ttw/HDYB4oizJS+qCv1ETIBhaOSjGRejuMvSP+1r4RJKIkFxOSIQ5iWW+JoMpsDxYdoF5H3l/0I4+B8pwvzrSByKjNh0iYUN1RtCIoXyO9CAjSANCDHC+ZE1BJMjv+0Zry1OA45lKSejzfzQA1SfD98DZlVogQJt4ogqGkxjfA9AKIIkyC+MCdii9MhAwhYfooS0VzrCz/onlbavP/ZL9v8MxdlcEEu3BSxB44g+ShJAOCrvmtgSrKffciwx3wCkkVmApTRudeHehTqxTBm2zAIRS6kNi2ijUN9HGo4xKNjNoohRx9TEAOOgMu8cCj33kGmNl8KRccVur0W+U3KClMtAAQO94hGTIKS4FEBWlqpVJY/Ggrud7xpswDXalAVSzddPmVRhAAHwg+ZJucAcc4I8LQyo16YpPb0mnu2f10CEMfAM3L7Re5RMlBykyVwBw1/AcLmX3HoAHVl+pmqhgWUPzEke/j9JRDPJka6XtsmNhRUljiBPLElpfNFagOyBNQC1rk+3L9H0qPeqwXpH3CuWGCkcJ8CLMglg/xQgtuoUj5suRlN/Wsz8i1IlE2PCag3rgj4yAf7f0XjKgoj7oKHMSNGQ2u6RN2DgK5mHCsHM8RiJxLLUEGwqnQrHI58+5b4HrZrsDqQKmWzMY0dMRBUxJGhYeklWynf/o5IEENFZGkHRUI4rAChoniKD7DOzCuOAcuZ7I0lnnuT21KlUdixY3ix4xxIP2zRSHpZ1UmLiIwKHFkqpH7UVZBWh4Sj2xN7U2OO9orbr7TulDu2tqVALzbnCtKBPs/AbT0MsgKiQEoHnC7PHH3sRBJAcmNNuIodBWMh/hIMwLA+ibBEhVjlnwEZ3PmN/44AO0KEfBQLAMSBP3T+mu9NHCYik0cj20z27FUi5ow0oExBUcF/YxQyeqWuv2TenmRRARc5/D55KKwSqcSWRqbnDt8iD16eIzcFkK6XNDu8BE4dVp1BLDtIgnYSsWLUnC85ZZN2ez2lYvOP2OdQ7Noqq1C9GzmohAK6xojW+3jRN/vnXY20pBzW0mrzmcluAscEX6+RVwi3JANgKVEpIFQbYTNVO3gYpLVQPAjU2oH0t/eASYSXh5TLqoiW0wSTe5BO8NmPQc1zuhZyCrewqE9z4T6NYJVIHinXB+3Oj4h1gfknE+l8s5aEmrjpRKCoyQbBvyXh97El2QWh5zN9aTwrne9BIc3ZYydLurO6gUg/scZ1OTxxXSucl1y9yn1Jm1Y5/fdcmKm9Qp5yviutBmDlw3bie0oyx3C/jixkyEs8zCl5Picda3qNvHxJ7p8GlfNDpkXxZPKAaSMaXUEGPpfWA8SV8wfZqnUiu2fbSiEDuVks1XFtG+Wrbd1vwGm0mX9+2Ujc0pdFjf+Crv1JkiNBge8IRI/p748khA8A+mTwXB5Zd/JBECgmXhoVuBjFamh/6id48NiqVioVnA1zclNlMx7KGBYW29mYCwPzrxp+NTElZCEAmNbbaeP729/Y8rYZceiqLi/MVOgWAJtTP4zMse8LxJns7SMEga0pM+255MQBMFO0HA/0zKRkSRiT6g4paq2YErlAquaB4U0olENYaUSw3pu0OwRSC8FVT54JweA7lPozP46w/p+xP0kL8KvJhUPcckZUU10D8b8A6kQIv3aCXSGR/sj3QCY8BK0D8hnzNNI+pKowLpaWjo7lt+IiDDylShJhwBIF1GtfMuFD1sNCQmM8onSxTfHtqwW7Q+iMWQTqceAk9Pi3v8N5Hl0qlYAHBHBYpvArbCy1kG6pwywVFLYGxyx0MbWSSvCfTGTYH8ykrwFs8BUmQAQCtJRs5gbiNtPHudhV0pywABVIKBuRG99voCmfCxtE3aVSLPV2KNiODbvBI8z78bw9DU95331NueDeJJmJrQFm35kvSZpb3sbc9tTEC5mDZtyqlBTY4u5EdQ5hU1WDFBujJjWruzfV3dKEj4uEOsLQjhMPq46xGeFuEHL3JaMWeNQDjcgjlO9OKytBptqupKujLdMiwXbqBLcUb6ivqtepphWl7aieKwyYyFZxEV6ERSDqXbVI2qnWyAmJNhmalBpJ5aC8iCmJ19qn374K6vphMPWj1I+M8cAJbHwB2uJqdfwHek7KpOe61IuZQ153+j+/761a9Wt9v9nMBGaJG3PGQgNGS82mXqtUGi+pGxyEY2HOQGfGrjDwEZQX9Y4/5i9qr2EArq/YDP664gb+jnz/Kl0Dgc4LmbEwNFKoTl906y86+3MOX9QqNv5xKPxJ+cxW/aNFEKc9Y38P+ABO8Pmenb2Gvj+Hif4pMmqNJiwOsmF8lsKQnCPXsMvZnQhPXyKjj8i8mAv4iOJwxrjjKMLmqCE7MLosHM8HMLoboPg8ysYgBQXHZKjNyY1KV2ekm6dlupN83OM5YY/4xAMP4AfS6qEIlma3oqZVqpdowaqaV7XdN/AoIwdj1eoV6WPmn7X/pMpVvJAzisSHOMWtKzCQrMDU1IG9tk20xJGQpCqVTbODYlstVk7X4cqjLVGGZKylK8vTVlF2Mjg6Xk0X3rmHKry9SFNB9cs3lAuMnbn7S4gK/et5opHzjJhO52WbJChJ4teS2fSBluw+bngKUGQeAzUgUYCQT6ZzBzua6JOQ3WqLps8E/8O0pqB3WnvIBcSpVqZpENcO8Tc/P5HkA9zTS28UBQct0oOcgpNgyh0/8uEDE6IITwLwmB1SIsyE8Dwfvoffvkb4ZJqM9gtkyOi6yO5tACn24/gFV45Ep6l6SvGyAPJx49vXdQSBz2Ux+txHUNRRMMhdGjKFXU51lKs3I8bwCblavVaHxOiOz6ftbaKAryOf2dVHkd6zOYJfoT7rKJeqFE3ZnHigXDpH4B/9ZpZdrSD3ltHwcaL2vG9f5fYrlZYHDus0ACPBOkos33EeRUkXACiV/5F9R7Sp4BpN3PQ+GBQEJJKGI2Bvm5wMBmekUttv6vXqzkeUKFnHiCdD4aNYOXHxeRlp0SuE2GiSYB3yiP/5OOf12g9dKlahrZIdYrnrt89+HGJw/2dlFHO51zlbp0B3EpC15ODZ0ePUiYpvXyc+ZinrqFmvVIh97kv2QavOOsBb56wrnYgqNh62vF6voj2/7Z2dn8IvPgr5wzCq1V2AcdkbvL+87HWHJ+dnOQhrJCjBcVlKPGJTJU3EhmIQihCH/bAkUyoyTqQwa3SoTFBCuXh70j8fnMNHjxz3BuRDp997fzkgOrnAaIlBsTsFdyg4FIaRy+5GxKTeLKSpBOZqC8nEPB1aSiWwgXTw7wZw8tti4PwdipEnC2OKuJL60NXHP4g0KY9puNkmMulfAVx6L3V5QBKfsjhj8+l99RYpJco+kn7msDxqpyTfdSmrssL/+ZP/+ZMf9idpu08RtVBQLgpAdWsSfmf4LKsQDIayduQEhIBNHMlHHDqm/Gbg4fqLeudF7TX8qC6PzuISdemf+Kyq72bDcHEhp1XhB2t5dVWqNEunv6p/pc/B5KVqpmAwx2K/BmYhrWL/JVAFeI2szDtNOz9r/V5xaZOnICXs04ATnSWXNU21oFKWtUX9Sd1y0rjFLq9UDPTKAossk90wF/wwPmgPqfP8S7YlsCZtj1rMcQJq4xPwEnOqXjkQi42AguoETOczAyfpTqtjM2uEWafN9BENJVIxvX++DyzCCsHumyqgjjIdPgJyYfYgDjAXvG9B+pjPfFEfnyviEVX13/qVkYowOWxpzajWtI85vjWlngeXJdv0jmmzGfO0fKs5E0291mxmyr9wU8pBGt3i3UXW+3wWsuUpK5hl5IKB/OMbGU5PaejdWXWH0VM6oRH3VpIXgfpKHKBSEHNjT+pP4t2SnnsqGIdlbfhcQVDIq+68X4vAZz4HvMljNuZCi+Xu/ZXNjYUHdRYhLNxaWPyAnrZ6tCgPYTNt1chbd8nL7i/0qNNGe2qCigZsoOXa+dwjxycfyPIxDfbzs8OU7OSmkJl9MmaGUVRUDxAl3fvlCfg4oBnFgTz3ued8Y+zLak02DOUVwNgrtk/xEWTfiwhkFBOIM5EMDyraZEjXYL1mI3wUGTED4qR/nUxQL3+Y+LBeMaEpefXlO+/0FTO/e+e1j/aKA7E7xG6sozZwR/jUVCKh3Hs4E9+fOIiaCPkKomGOHOpdtV+jqySxxwjkp5ALwq865HkjF+Rxb+QE4HDmIbL0PG6Re0QY8cmoBGEgY7VUKiGayHdimQeD8QJm9fgrJZAJ25wmR4MwtZjSqSlCtTZgk9RkaiI7ikmSTwqLGllOtA41Up4qFeYmp471iu3lcz65Z+pkEYPT6oPJpZ5lMagWl606f0KomsxFl3nxMQv9GFgRxfazAiGHDl95Uldsdywrhuw3a57emf2fIrbE8OhtlD9QlVr9TL69hXjToxY/jkD/42J6S0FKpFtU0s3v351XUi2wHkDGwH1HmIUU2105Qgbf/p6FCm1G6AMR4ON9JW4F+H4rXpdLNuzIiQdZmIghIQNN8B2enBLLmuSfYBnFpdhybh+EpH+LgXIRBzappfSXtmE91kT5V1j8artevcFlfE8WX2zNacUbNZRbJMnL6V56tcIMpILLty828oYwogCfxubyWXZ+aXbOL81Irb8//GDCaOLx3+bWwUa5nPmV8PnzLKoQSSc+Jqooxtf38ArfWPx/bYjDo5w+AAA="} diff --git a/tests/fake_warc_server.py b/tests/fake_warc_server.py new file mode 100644 index 0000000..b4b4754 --- /dev/null +++ b/tests/fake_warc_server.py @@ -0,0 +1,42 @@ +"""WARC files served by fake HTTP range requests, with realistic responses (206, Content-Range, exact bytes).""" +from io import BytesIO +from typing import Dict, List +from unittest.mock import Mock + +from warcio import WARCWriter + +from cdx_toolkit.repackage.warc_records import get_bytes_from_warc_record + + +def warc_record_bytes(url: str) -> bytes: + record = WARCWriter(None).create_warc_record(url, 'resource', payload=BytesIO(b'content of ' + url.encode())) + return get_bytes_from_warc_record(record, gzip=True) + + +class FakeWarcServer: + """WARC files (filename -> bytes) served under `prefix`; `get` stands in for myrequests_get.""" + + def __init__(self, prefix: str = 'https://data.example.org'): + self.prefix = prefix + self.files: Dict[str, bytes] = {} + self.requests: List[str] = [] + + def add_captures(self, filename: str, urls: List[str]) -> List[dict]: + """Append a WARC record per URL to the file, return the captures (CDX fields) of the records.""" + captures = [] + for url in urls: + data = warc_record_bytes(url) + offset = len(self.files.get(filename, b'')) + self.files[filename] = self.files.get(filename, b'') + data + captures.append({'url': url, 'filename': filename, 'offset': str(offset), 'length': str(len(data))}) + return captures + + def get(self, url, headers=None, **kwargs): + self.requests.append(url) + data = self.files.get(url[len(self.prefix):].lstrip('/')) + if data is None: + return Mock(status_code=404, headers={}, content=b'not found') + start, end = (int(n) for n in headers['Range'][len('bytes='):].split('-')) + body = data[start:end + 1] + content_range = f'bytes {start}-{start + len(body) - 1}/{len(data)}' + return Mock(status_code=206, headers={'Content-Range': content_range}, content=body) diff --git a/tests/repackage/__init__.py b/tests/repackage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/repackage/test_hf_live.py b/tests/repackage/test_hf_live.py new file mode 100644 index 0000000..dc20cab --- /dev/null +++ b/tests/repackage/test_hf_live.py @@ -0,0 +1,126 @@ +"""Live tests of the Hugging Face Storage Buckets (CDXT_LIVE_TESTS=hf): `cdxt index cc-url-index-hf`, and +`cdxt repackage` reading from hf://buckets/commoncrawl/commoncrawl and writing to TEST_HF_BUCKET. + +DuckDB reads of the columnar index over hf:// are slow (minutes for a single parquet file, mostly the latency +of the Hub API), so the index test reads a single file with LIMIT, and the repackage tests use known captures. +""" +import json +import os + +import fsspec +import pytest +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main +from tests.conftest import requires_duckdb, requires_live_hf + + +HF_PREFIX = 'hf://buckets/commoncrawl/commoncrawl' +CRAWL = 'CC-MAIN-2026-17' +INDEX_FILE = (f'{HF_PREFIX}/cc-index/table/cc-main/warc/crawl={CRAWL}/subset=warc/' + 'part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet') + +# captures of CC-MAIN-2026-17 (warc and crawldiagnostics files), from the CDX API +CAPTURES = [ + {'url': 'https://commoncrawl.org/', 'timestamp': '20260410082607', 'status': '200', + 'digest': 'ZVJ34E3JKBK2AXINWBT3QWHBDNYZU32Z', 'offset': '133904328', 'length': '7487', + 'filename': 'crawl-data/CC-MAIN-2026-17/segments/1775805908305.14/warc/' + 'CC-MAIN-20260410081153-20260410111153-00389.warc.gz'}, + {'url': 'https://www.commoncrawl.org/', 'timestamp': '20260413052322', 'status': '200', + 'digest': 'CAZVHFEZWLAZELACXKKCVDGSKTUQCRJ7', 'offset': '597179815', 'length': '7496', + 'filename': 'crawl-data/CC-MAIN-2026-17/segments/1775805908721.20/warc/' + 'CC-MAIN-20260413040425-20260413070425-00340.warc.gz'}, + {'url': 'http://commoncrawl.org/', 'timestamp': '20260410082607', 'status': '301', + 'digest': 'LGGT76I5HYSS6HQT36GPANELE4H7FWR7', 'offset': '157143', 'length': '877', + 'filename': 'crawl-data/CC-MAIN-2026-17/segments/1775805908305.14/crawldiagnostics/' + 'CC-MAIN-20260410081153-20260410111153-00494.warc.gz'}, +] + + +def _write_captures(tmpdir): + path = os.path.join(str(tmpdir), 'captures.jsonl') + with open(path, 'w') as f: + for capture in CAPTURES: + f.write(json.dumps(capture) + '\n') + return path + + +def _responses(path): + """(WARC-Target-URI, WARC-Payload-Digest) of the response records of a WARC file (sorted: the records are + written in the order of their WARC files and offsets).""" + with fsspec.open(path, 'rb') as stream: + return sorted((record.rec_headers.get_header('WARC-Target-URI'), + record.rec_headers.get_header('WARC-Payload-Digest')) + for record in ArchiveIterator(stream) if record.rec_type == 'response') + + +def _expected(): + return sorted((c['url'], 'sha1:' + c['digest']) for c in CAPTURES) + + +@requires_live_hf +@requires_duckdb +def test_index_cc_url_index_hf_sql(tmpdir): + sql = os.path.join(str(tmpdir), 'query.sql') + with open(sql, 'w') as f: + f.write(f""" + SELECT url, warc_filename, warc_record_offset, warc_record_length + FROM read_parquet('{INDEX_FILE}') + LIMIT 3 + """) + captures_path = os.path.join(str(tmpdir), 'captures.jsonl') + main(args=['index', 'cc-url-index-hf', '--sql', sql, '--yes', '-o', captures_path]) + + with open(captures_path) as f: + captures = [json.loads(line) for line in f] + assert len(captures) == 3 + assert all(c['filename'].startswith(f'crawl-data/{CRAWL}/') for c in captures) + assert all(int(c['length']) > 0 for c in captures) + + +@requires_live_hf +def test_repackage_from_hf(tmpdir): + main(args=['repackage', _write_captures(tmpdir), f'--prefix={tmpdir}/TEST_hf', '--warc-download-prefix', + HF_PREFIX]) + + assert _responses(os.path.join(str(tmpdir), 'TEST_hf-001.warc.gz')) == _expected() + + +@requires_live_hf +def test_repackage_to_hf_bucket(tmpdir, hf_tmpdir): + main(args=['repackage', _write_captures(tmpdir), f'--prefix={hf_tmpdir}/TEST_hf', '--warc-download-prefix', + HF_PREFIX]) + + assert _responses(f'{hf_tmpdir}/TEST_hf-001.warc.gz') == _expected() + + +@requires_live_hf +def test_repackage_from_hf_parallel_readers(tmpdir): + main(args=['repackage', _write_captures(tmpdir), f'--prefix={tmpdir}/TEST_hf', '--warc-download-prefix', + HF_PREFIX, '--parallel-readers', '3']) + + assert _responses(os.path.join(str(tmpdir), 'TEST_hf-001.warc.gz')) == _expected() + + +@requires_live_hf +def test_index_output_to_hf_bucket(tmpdir, hf_tmpdir, capsys): + """cdxt index writes to a bucket: a single output, and one output per CDX file, which needs a new or empty + directory (also in a bucket).""" + cdx = os.path.join(str(tmpdir), 'in', 'index.cdx') + os.makedirs(os.path.dirname(cdx)) + with open(cdx, 'w') as f: + for capture in CAPTURES: + f.write(f'org,commoncrawl)/ {capture["timestamp"]} {json.dumps(capture)}\n') + + main(args=['index', cdx, '--domain', 'commoncrawl.org', '-o', f'{hf_tmpdir}/captures.jsonl']) + with fsspec.open(f'{hf_tmpdir}/captures.jsonl', 'rt') as f: + assert [json.loads(line)['url'] for line in f] == [c['url'] for c in CAPTURES] + + main(args=['index', os.path.join(str(tmpdir), 'in', '*.cdx'), '--domain', 'commoncrawl.org', + '-o', f'{hf_tmpdir}/filtered']) + with fsspec.open(f'{hf_tmpdir}/filtered/index.cdx', 'rt') as f: + assert len(f.read().splitlines()) == len(CAPTURES) + with pytest.raises(SystemExit): + main(args=['index', os.path.join(str(tmpdir), 'in', '*.cdx'), '--domain', 'commoncrawl.org', + '-o', f'{hf_tmpdir}/filtered']) + assert 'is not empty' in capsys.readouterr().err diff --git a/tests/repackage/test_hf_reader.py b/tests/repackage/test_hf_reader.py new file mode 100644 index 0000000..c0eda4a --- /dev/null +++ b/tests/repackage/test_hf_reader.py @@ -0,0 +1,276 @@ +"""Offline unit tests for hf:// bucket URL helpers and the HF reader (no network).""" +import asyncio +import time +from unittest.mock import patch + +import pytest + +from cdx_toolkit.repackage.readers.hf import ( + HfRangeReader, + make_hf_reader, + signed_url_expiry, + hf_fs_path, + hf_resolve_url, + retry_after_seconds, +) +from cdx_toolkit.repackage.urls import is_hf_url + + +def test_is_hf_url(): + assert is_hf_url('hf://buckets/commoncrawl/commoncrawl/crawl-data/x.warc.gz') + assert not is_hf_url('s3://commoncrawl/crawl-data/x.warc.gz') + assert not is_hf_url('https://data.commoncrawl.org/x.warc.gz') + assert not is_hf_url('') + assert not is_hf_url(None) + + +def test_hf_fs_path_strips_scheme(): + assert hf_fs_path('hf://buckets/ns/name/a/b.warc.gz') == 'buckets/ns/name/a/b.warc.gz' + + +def test_hf_fs_path_rejects_non_hf(): + with pytest.raises(ValueError): + hf_fs_path('s3://bucket/key') + + +def test_hf_resolve_url_maps_bucket_to_resolve(): + url = 'hf://buckets/commoncrawl/commoncrawl/crawl-data/CC-MAIN-2026-21/seg/x.warc.gz' + assert hf_resolve_url(url) == ( + 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/' + 'crawl-data/CC-MAIN-2026-21/seg/x.warc.gz' + ) + + +def test_hf_resolve_url_rejects_non_bucket(): + # only hf://buckets/// is supported for resolve mapping + with pytest.raises(ValueError): + hf_resolve_url('hf://datasets/foo/bar/x.warc.gz') + with pytest.raises(ValueError): + hf_resolve_url('hf://buckets/onlytwo/segments') + + +def test_retry_after_prefers_retry_after_header(): + assert retry_after_seconds({'Retry-After': '12'}, default=99) == 12.0 + + +def test_retry_after_parses_ratelimit_t_field(): + h = {'RateLimit': '"resolvers";r=0;t=287'} + assert retry_after_seconds(h, default=99) == 287.0 + + +def test_retry_after_falls_back_to_default(): + assert retry_after_seconds({}, default=7.5) == 7.5 + # malformed Retry-After falls through to default (no RateLimit present) + assert retry_after_seconds({'Retry-After': 'soon'}, default=3.0) == 3.0 + + +# --- HfRangeReader: resolve each WARC file once, read the records from the signed CDN URL --- + +WARC_A = 'hf://buckets/commoncrawl/commoncrawl/crawl-data/a.warc.gz' +WARC_B = 'hf://buckets/commoncrawl/commoncrawl/crawl-data/b.warc.gz' +DATA = bytes(range(256)) * 4 + + +class FakeResponse: + def __init__(self, status, headers=None, body=b''): + self.status = status + self.headers = headers or {} + self._body = body + + async def read(self): + return self._body + + async def release(self): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + +class FakeSession: + """Resolve URLs redirect to a signed CDN URL; the CDN serves ranges of DATA.""" + + def __init__(self, ttl=3600, head_statuses=(), get_statuses=()): + self.ttl = ttl + self.head_statuses = list(head_statuses) # statuses returned before the redirect, e.g. 429 + self.get_statuses = list(get_statuses) # statuses returned before serving data, e.g. 403 + self.heads, self.gets = [], [] + + def head(self, url, headers=None, allow_redirects=True): + assert not allow_redirects + self.heads.append((url, headers)) + if self.head_statuses: + return FakeResponse(self.head_statuses.pop(0), {'Retry-After': '0'}) + expires = int(time.time() + self.ttl) + location = f'https://cdn.example/{url.rsplit("/", 1)[-1]}?Expires={expires}&n={len(self.heads)}' + return FakeResponse(302, {'Location': location}) + + def get(self, url, headers=None): + self.gets.append((url, headers)) + if self.get_statuses: + return FakeResponse(self.get_statuses.pop(0)) + start, end = map(int, headers['Range'][len('bytes='):].split('-')) + body = DATA[start:end + 1] + return FakeResponse(206, {'Content-Range': f'bytes {start}-{start + len(body) - 1}/{len(DATA)}'}, body) + + +def _read(reader, url, offset, length): + return reader.ranged_get(url, offset, length, max_attempts=3, base_backoff_seconds=0) + + +def test_signed_url_expiry(): + assert signed_url_expiry('https://cdn.example/x?Expires=1791445181&Signature=s', 600) == 1791445181 + assert abs(signed_url_expiry('https://cdn.example/x', 600) - (time.time() + 600)) < 5 + + +def test_resolves_each_file_once(): + session = FakeSession() + reader = HfRangeReader(session, token='tok') + + async def run(): + return [await _read(reader, url, o, 10) for url, o in [(WARC_A, 0), (WARC_A, 100), (WARC_B, 5), (WARC_A, 50)]] + + assert asyncio.run(run()) == [DATA[0:10], DATA[100:110], DATA[5:15], DATA[50:60]] + assert [url for url, _ in session.heads] == [ + 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/crawl-data/a.warc.gz', + 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/crawl-data/b.warc.gz', + ] + assert session.heads[0][1] == {'Authorization': 'Bearer tok'} + # the token is not sent to the CDN, the URL is signed + assert all(url.startswith('https://cdn.example/') and 'Authorization' not in h for url, h in session.gets) + assert (reader.resolve_count, reader.read_count) == (2, 4) + + +def test_concurrent_reads_share_one_resolve(): + session = FakeSession() + reader = HfRangeReader(session) + + async def run(): + return await asyncio.gather(*[_read(reader, WARC_A, o, 4) for o in range(0, 80, 8)]) + + assert asyncio.run(run()) == [DATA[o:o + 4] for o in range(0, 80, 8)] + assert len(session.heads) == 1 + + +def test_resolves_again_before_expiry(): + session = FakeSession(ttl=HfRangeReader.EXPIRY_MARGIN - 1) # signed URLs that are about to expire + reader = HfRangeReader(session) + + async def run(): + await _read(reader, WARC_A, 0, 4) + await _read(reader, WARC_A, 4, 4) + + asyncio.run(run()) + assert len(session.heads) == 2 + + +def test_resolves_again_when_cdn_rejects_signed_url(): + session = FakeSession(get_statuses=[403]) + reader = HfRangeReader(session) + + assert asyncio.run(_read(reader, WARC_A, 10, 4)) == DATA[10:14] + assert len(session.heads) == 2 # the rejected URL was resolved again + assert session.gets[-1][0].endswith('&n=2') + + +def test_waits_on_resolver_rate_limit(): + session = FakeSession(head_statuses=[429, 429]) + reader = HfRangeReader(session) + + async def no_sleep(_): + return None + + with patch('cdx_toolkit.repackage.readers.hf.asyncio.sleep', side_effect=no_sleep): + assert asyncio.run(_read(reader, WARC_A, 0, 4)) == DATA[0:4] + + assert reader.throttle_count == 2 + assert reader.resolve_count == 1 + + +def test_resolve_without_redirect_reads_from_the_resolve_url(): + session = FakeSession(head_statuses=[200]) + reader = HfRangeReader(session, token='tok') + + assert asyncio.run(_read(reader, WARC_A, 0, 4)) == DATA[0:4] + url, headers = session.gets[0] + assert url.startswith('https://huggingface.co/buckets/') and headers['Authorization'] == 'Bearer tok' + + +def test_resolve_errors_are_retried_then_raised(): + # resolving is retried within each read attempt (3 x 3 attempts) + session = FakeSession(head_statuses=[500] * 3) + assert asyncio.run(_read(HfRangeReader(session), WARC_A, 0, 4)) == DATA[0:4] + + session = FakeSession(head_statuses=[500] * 9) + with pytest.raises(OSError, match='HF HEAD .* -> 500'): + asyncio.run(_read(HfRangeReader(session), WARC_A, 0, 4)) + assert len(session.heads) == 9 + + +@pytest.mark.parametrize( + 'get_statuses, offset, length, error', + [ + ([500, 500, 500], 0, 4, 'HF GET .* -> 500'), + ([], 1000, 100, 'requested bytes 1000-1099, got Content-Range'), # short read + ], +) +def test_read_errors_are_retried_then_raised(get_statuses, offset, length, error): + session = FakeSession(get_statuses=get_statuses) + with pytest.raises(OSError, match=error): + asyncio.run(_read(HfRangeReader(session), WARC_A, offset, length)) + assert len(session.gets) == 3 + + +def test_cdn_rate_limit_is_waited_out(caplog): + session = FakeSession(get_statuses=[429]) + reader = HfRangeReader(session) + + async def run(): + data = await _read(reader, WARC_A, 0, 4) + await reader.aclose() + return data + + with caplog.at_level('INFO', logger='cdx_toolkit.repackage.readers.hf'): + assert asyncio.run(run()) == DATA[0:4] + assert reader.throttle_count == 1 + assert 'HF reader: 1 records read with 1 resolve requests' in caplog.text + assert 'HF resolver throttling: 1 x 429' in caplog.text + + +def test_gives_up_after_too_many_rate_limits(monkeypatch): + monkeypatch.setattr(HfRangeReader, 'MAX_THROTTLE_RETRIES', 0) + session = FakeSession(get_statuses=[429, 429, 429]) + with pytest.raises(OSError, match='HF 429: gave up after'): + asyncio.run(_read(HfRangeReader(session), WARC_A, 0, 4)) + + +def test_signed_url_cache_is_bounded(monkeypatch): + monkeypatch.setattr(HfRangeReader, 'MAX_CACHED_URLS', 1) + session = FakeSession() + reader = HfRangeReader(session) + + async def run(): + for url in (WARC_A, WARC_B, WARC_A): + await _read(reader, url, 0, 4) + + asyncio.run(run()) + assert len(session.heads) == 3 # WARC_A was evicted by WARC_B + assert list(reader._signed_urls) == [ + 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/crawl-data/a.warc.gz', + ] + + +def test_make_hf_reader(): + pytest.importorskip('aiohttp') + pytest.importorskip('huggingface_hub') + + async def run(): + with patch('huggingface_hub.get_token', return_value='tok'): + async with make_hf_reader() as reader: + return reader + + reader = asyncio.run(run()) + assert isinstance(reader, HfRangeReader) and reader.token == 'tok' diff --git a/tests/repackage/test_jobs.py b/tests/repackage/test_jobs.py new file mode 100644 index 0000000..b7d3673 --- /dev/null +++ b/tests/repackage/test_jobs.py @@ -0,0 +1,73 @@ +import pytest + +from cdx_toolkit.repackage.jobs import InvalidWaybackCapture, WaybackSource, join_warc_url + +IA_CAPTURE = {'url': 'http://example.com/a b', 'timestamp': '20240101120000', 'status': '200'} + + +def test_join_warc_url_relative(): + assert ( + join_warc_url('https://data.commoncrawl.org', 'crawl-data/x.warc.gz') + == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' + ) + + +def test_join_warc_url_no_double_slash(): + assert ( + join_warc_url('https://data.commoncrawl.org/', 'crawl-data/x.warc.gz') + == 'https://data.commoncrawl.org/crawl-data/x.warc.gz' + ) + assert join_warc_url('s3://bucket', '/crawl-data/x.warc.gz') == 's3://bucket/crawl-data/x.warc.gz' + + +def test_join_warc_url_empty_prefix(): + assert join_warc_url('', 'crawl-data/x.warc.gz') == 'crawl-data/x.warc.gz' + assert join_warc_url(None, 'crawl-data/x.warc.gz') == 'crawl-data/x.warc.gz' + + +def test_join_warc_url_absolute_filename_passthrough(): + assert join_warc_url('https://data.commoncrawl.org', 's3://cc/x.warc.gz') == 's3://cc/x.warc.gz' + assert join_warc_url('', 'https://host/x.warc.gz') == 'https://host/x.warc.gz' + + +def wayback_job(record, wb='ia'): + return next(WaybackSource([record], wb).iter_range_jobs()) + + +def test_wayback_job(): + job = wayback_job(dict(IA_CAPTURE, status=200, mime='text/html')) + assert job.url == 'https://web.archive.org/web/20240101120000id_/http%3A//example.com/a%20b' + assert job.wb == 'https://web.archive.org/web' + assert job.capture == {**IA_CAPTURE, 'mime': 'text/html'} # the status as a string, like the CDX API's + assert wayback_job(IA_CAPTURE, 'https://wb.example.org/web').url.startswith('https://wb.example.org/web/') + + +def test_wayback_jobs_from_another_wayback_with_warc_location(): + """pywb servers return filename/offset/length too, relative to their own storage: replayed, not copied.""" + capture = dict(IA_CAPTURE, filename='collections/my/archive/x.warc.gz', offset='0', length='10') + assert wayback_job(capture, 'https://wb.example.org/web').capture == capture + + +@pytest.mark.parametrize('filename', [ + 'crawl-data/CC-MAIN-2024-33/segments/1/warc/x.warc.gz', + 'https://data.commoncrawl.org/crawl-data/CC-NEWS/2024/01/x.warc.gz', + 's3://commoncrawl/crawl-data/CC-MAIN-2024-33/x.warc.gz', +]) +def test_wayback_rejects_common_crawl_captures(filename): + with pytest.raises(InvalidWaybackCapture, match='Common Crawl capture .* do not use --wb'): + wayback_job(dict(IA_CAPTURE, filename=filename, offset='0', length='10')) + + +@pytest.mark.parametrize('change, error', [ + ({'url': None}, 'has no url'), + ({'timestamp': ''}, 'has no timestamp'), + ({'status': None}, 'has no status'), + ({'url': 'example.com/a'}, 'not an http.s. URL'), + ({'url': 'ftp://example.com/a'}, 'not an http.s. URL'), + ({'url': 'http://'}, 'not an http.s. URL'), + ({'timestamp': '2024-01-01'}, 'invalid timestamp'), + ({'timestamp': '202'}, 'invalid timestamp'), +]) +def test_wayback_rejects_invalid_captures(change, error): + with pytest.raises(InvalidWaybackCapture, match=error): + wayback_job({**IA_CAPTURE, **change}) diff --git a/tests/repackage/test_local_output.py b/tests/repackage/test_local_output.py new file mode 100644 index 0000000..dcc17ba --- /dev/null +++ b/tests/repackage/test_local_output.py @@ -0,0 +1,73 @@ +import asyncio +import os + +import pytest + +from cdx_toolkit.repackage.writers.local import LocalOutputFile + + +def run(coro): + return asyncio.run(coro) + + +def test_writes_buffered_and_byte_exact(tmp_path): + path = tmp_path / 'out.warc.gz' + chunks = [bytes(range(256)), b'small', b'x' * 20000, b'', b'\x00\xff' * 10] + + async def write(): + writer = LocalOutputFile(str(path), buffer_size=1024) + await writer.start() + await writer.write(chunks[1]) + assert path.read_bytes() == b'' # buffered + await writer.write(chunks[2]) + assert len(path.read_bytes()) == len(chunks[1] + chunks[2]) # flushed at the buffer size + for chunk in chunks[3:]: + await writer.write(chunk) + await writer.close() + + run(write()) + assert path.read_bytes() == b''.join(chunks[1:]) + + +def test_close_without_start(tmp_path): + run(LocalOutputFile(str(tmp_path / 'out')).close()) + assert not (tmp_path / 'out').exists() + + +def test_abort_removes_the_file(tmp_path): + path = tmp_path / 'out.warc.gz' + + async def write_and_abort(): + writer = LocalOutputFile(str(path), buffer_size=4) + await writer.start() + await writer.write(b'incomplete') + await writer.abort() + await writer.abort() # idempotent + + run(write_and_abort()) + assert not path.exists() + run(LocalOutputFile(str(path)).abort()) # without start + + +def test_start_fails_without_the_directory(tmp_path): + with pytest.raises(FileNotFoundError): + run(LocalOutputFile(os.path.join(str(tmp_path), 'missing', 'out')).start()) + + +def test_close_removes_the_file_if_the_last_write_fails(tmp_path): + async def close_with_failing_flush(): + writer = LocalOutputFile(str(tmp_path / 'out')) + await writer.start() + await writer.write(b'x') + handle = writer.file_handle + + async def fail(): + raise OSError('disk full') + + writer._flush = fail + with pytest.raises(OSError): + await writer.close() + assert writer.file_handle is None and handle.closed + + run(close_with_failing_flush()) + assert not (tmp_path / 'out').exists() diff --git a/tests/repackage/test_multiprocess_cancel.py b/tests/repackage/test_multiprocess_cancel.py new file mode 100644 index 0000000..95edfc4 --- /dev/null +++ b/tests/repackage/test_multiprocess_cancel.py @@ -0,0 +1,55 @@ +"""A failing worker process stops the others (real spawned processes).""" +import multiprocessing +import os +import time + +import pytest + +from cdx_toolkit.repackage import multiprocess +from cdx_toolkit.repackage.multiprocess import run_worker_processes + + +def worker(config): + """'fail': fails after a moment; 'wait': waits for the cancel event, then stops (or completes after 20 s); + 'ignore': does not look at the cancel event.""" + kind, marker = config + if kind == 'fail': + time.sleep(0.2) + raise OSError('disk full') + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + if kind == 'wait' and multiprocess._CANCEL.is_set(): + return 'stopped' + time.sleep(0.05) + with open(marker, 'w') as f: + f.write('completed') + return 'completed' + + +@pytest.mark.parametrize('failing_first', [True, False]) +def test_a_failing_worker_stops_the_others(tmp_path, failing_first): + configs = [('wait', str(tmp_path / 'a')), ('wait', str(tmp_path / 'b'))] + configs.insert(0 if failing_first else len(configs), ('fail', None)) + ctx = multiprocessing.get_context('spawn') + start = time.monotonic() + with pytest.raises(OSError, match='disk full'): + run_worker_processes(worker, configs, ctx.Value('i', 0), ctx, cancel_timeout=15) + assert time.monotonic() - start < 15 # the others did not run to completion (20 s) + assert os.listdir(tmp_path) == [] + + +def test_a_worker_that_does_not_stop_is_terminated(tmp_path): + configs = [('fail', None), ('ignore', str(tmp_path / 'a'))] + ctx = multiprocessing.get_context('spawn') + start = time.monotonic() + with pytest.raises(OSError, match='disk full'): + run_worker_processes(worker, configs, ctx.Value('i', 0), ctx, cancel_timeout=1) + assert time.monotonic() - start < 15 + time.sleep(0.5) + assert os.listdir(tmp_path) == [] + + +def test_results_in_the_order_of_the_configs(tmp_path): + ctx = multiprocessing.get_context('spawn') + file_sequence = ctx.Value('i', 0) + assert run_worker_processes(abs, [-3, 1, -2], file_sequence, ctx) == [3, 1, 2] diff --git a/tests/repackage/test_multiprocess_naming.py b/tests/repackage/test_multiprocess_naming.py new file mode 100644 index 0000000..5ac1440 --- /dev/null +++ b/tests/repackage/test_multiprocess_naming.py @@ -0,0 +1,51 @@ +"""Unit tests for multi-process output naming: the file numbers shared by all worker +processes, and copying staged files to their (non-S3) destination. + +No S3 or fetching. +""" +import multiprocessing +import os + +from cdx_toolkit.repackage import multiprocess +from cdx_toolkit.repackage.multiprocess import _copy_staged_files +from cdx_toolkit.repackage.writers.series import generate_warc_filename + + +def test_warc_filenames(): + # the formatter of the file names, also for the log line of the multi-process mode + assert generate_warc_filename('/out/run', 1, gzip=True) == '/out/run-001.warc.gz' + assert generate_warc_filename('/out/run', 12, gzip=True) == '/out/run-012.warc.gz' + assert generate_warc_filename('/out/run', 1234, gzip=True) == '/out/run-1234.warc.gz' + assert generate_warc_filename('/out/run', 2, writer_subprefix='SUB', gzip=True) == '/out/run-SUB-002.warc.gz' + assert generate_warc_filename('s3://b/p/run', 3, gzip=True) == 's3://b/p/run-003.warc.gz' + assert generate_warc_filename('hf://buckets/a/b/run', 3) == 'hf://buckets/a/b/run-003.warc' + + +def _take_numbers(n): + return [multiprocess._next_file_sequence(0) for _ in range(n)] + + +def test_file_numbers_are_shared_across_processes(): + ctx = multiprocessing.get_context() + counter = ctx.Value('i', 0) + results = multiprocess.run_worker_processes(_take_numbers, [5, 5, 5], counter, ctx) + numbers = [n for result in results for n in result] + + # every file number is used exactly once, without gaps + assert sorted(numbers) == list(range(1, 16)) + assert counter.value == 15 + + +def test_copy_staged_files(tmp_path): + staging = tmp_path / 'staging' + staging.mkdir() + for name in ('warc-001.warc.gz', 'warc-002.warc.gz', 'other.txt'): + (staging / name).write_bytes(name.encode()) + dest_dir = tmp_path / 'dest' + dest_dir.mkdir() + + copied = _copy_staged_files(str(staging / 'warc'), str(dest_dir / 'RUN')) + + assert copied == [str(dest_dir / 'RUN-001.warc.gz'), str(dest_dir / 'RUN-002.warc.gz')] + assert sorted(os.listdir(dest_dir)) == ['RUN-001.warc.gz', 'RUN-002.warc.gz'] + assert (dest_dir / 'RUN-002.warc.gz').read_bytes() == b'warc-002.warc.gz' diff --git a/tests/repackage/test_producer.py b/tests/repackage/test_producer.py new file mode 100644 index 0000000..de3f31d --- /dev/null +++ b/tests/repackage/test_producer.py @@ -0,0 +1,31 @@ +import glob +from unittest.mock import patch + +import pytest + +from cdx_toolkit.repackage.jobs import RangeJob +from cdx_toolkit.repackage.repackager import Repackager +from cdx_toolkit.repackage.readers.http import HttpRangeReader + + +class FailingSource: + def iter_range_jobs(self): + yield RangeJob(url='https://data.commoncrawl.org/0.warc.gz', offset=0, length=1) + raise RuntimeError('boom') + + +def test_failing_source_fails_the_pipeline(tmp_path, caplog): + """A source that raises mid-iteration stops the readers and the writer, the error propagates.""" + async def read(self, job): + return b'x' + + repackager = Repackager( + source=FailingSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, n_parallel_readers=3, + ) + with patch.object(HttpRangeReader, 'read', read), pytest.raises(RuntimeError, match='boom'), \ + caplog.at_level('INFO', logger='cdx_toolkit.repackage.repackager'): + repackager.run() + assert glob.glob(str(tmp_path / 'out-*')) == [] + # the writer was stopped, it did not fail + assert 'WARC writer failed' not in caplog.text + assert 'WARC writer stopped' in caplog.text diff --git a/tests/repackage/test_readers.py b/tests/repackage/test_readers.py new file mode 100644 index 0000000..d6e2f51 --- /dev/null +++ b/tests/repackage/test_readers.py @@ -0,0 +1,70 @@ +import asyncio +import contextlib +from unittest.mock import patch + +import pytest + +from cdx_toolkit.repackage.jobs import RangeJob, WaybackJob +from cdx_toolkit.repackage.readers import Readers +from cdx_toolkit.repackage.readers.http import HttpRangeReader, RangeError, check_range_response +from cdx_toolkit.repackage.readers.wayback import WaybackReader + +from tests.conftest import requires_aioboto3 + + +@requires_aioboto3 +def test_readers_by_job(): + from cdx_toolkit.repackage.readers.s3 import S3RangeReader + + hf_reader = object() + + @contextlib.asynccontextmanager + async def fake_make_hf_reader(max_attempts, base_backoff_seconds): + assert (max_attempts, base_backoff_seconds) == (3, 0.1) + yield hf_reader + + async def main(): + async with contextlib.AsyncExitStack() as stack: + readers = Readers(stack, lambda: None, max_attempts=3, base_backoff_seconds=0.1) + + def for_url(url): + return readers.for_job(RangeJob(url, 0, 1)) + + http = await for_url('https://data.commoncrawl.org/a.warc.gz') + s3 = await for_url('s3://commoncrawl/a.warc.gz') + hf = await for_url('hf://buckets/commoncrawl/commoncrawl/a.warc.gz') + # one reader per scheme + assert await for_url('s3://commoncrawl/b.warc.gz') is s3 + assert await for_url('http://example.org/b.warc.gz') is http + # a wayback replay, by the type of the job (its replay URL is an http URL) + wb = 'https://web.archive.org/web' + wayback = await readers.for_job(WaybackJob(wb + '/1id_/http://example.com/', {}, wb)) + return http, s3, hf, wayback + + with patch('aioboto3.Session'), patch('cdx_toolkit.repackage.readers.hf.make_hf_reader', fake_make_hf_reader): + http, s3, hf, wayback = asyncio.run(main()) + assert isinstance(http, HttpRangeReader) + assert isinstance(wayback, WaybackReader) + assert isinstance(s3, S3RangeReader) and (s3.max_attempts, s3.base_backoff_seconds) == (3, 0.1) + assert hf is hf_reader + + +@pytest.mark.parametrize('status, content_range, body_length, offset, valid', [ + (206, 'bytes 10-19/100', 10, 10, True), + (206, 'bytes 10-19/*', 10, 10, True), + (200, None, 10, 0, True), # the whole file is the requested range + (200, None, 100, 10, False), # range ignored + (206, 'bytes 11-20/100', 10, 10, False), # wrong range + (206, 'bytes 10-19/100', 9, 10, False), # truncated body + (206, None, 10, 10, False), + (302, None, 0, 10, False), # redirects are not followed +]) +def test_check_range_response(status, content_range, body_length, offset, valid): + def check(): + check_range_response(status, content_range, body_length, offset, 10, 'https://example.org/x') + + if valid: + check() + else: + with pytest.raises(RangeError): + check() diff --git a/tests/repackage/test_repackage_cli.py b/tests/repackage/test_repackage_cli.py new file mode 100644 index 0000000..e8924f2 --- /dev/null +++ b/tests/repackage/test_repackage_cli.py @@ -0,0 +1,260 @@ +"""End-to-end tests of `cdxt repackage` with the async WARC fetcher. + +The captures are the first records of a Common Crawl CDX file (a CDXJ file); the WARC +records are fetched from data.commoncrawl.org or s3://commoncrawl, which are mocked with +recorded records (see tests/conftest.py). +""" +import gzip +import multiprocessing +import os +from typing import List, Optional + +import fsspec +import pytest +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main +from cdx_toolkit.record_io import parse_cdxj_line, read_records, write_records + +from tests.conftest import InlineExecutor, requires_aws_s3, TEST_DATA_PATH + + +pytestmark = pytest.mark.usefixtures('mock_data_commoncrawl') + + +CDX_PATH = TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz' +METADATA_RECORD_PATH = TEST_DATA_PATH / 'repackage/url_list.txt' + + +def write_captures(tmpdir, n: int = 10, name: str = 'captures.cdxj', s3_env=None) -> str: + """The first n captures of the CDX fixture (with their offsets in the mocked s3://commoncrawl, if given).""" + path = os.path.join(str(tmpdir), name) + with gzip.open(CDX_PATH, 'rt') as f_in: + lines = [f_in.readline() for _ in range(n)] + if s3_env: + write_records([s3_env.capture(parse_cdxj_line(line)) for line in lines], path, 'cdxj') + else: + with open(path, 'w') as f_out: + f_out.writelines(lines) + return path + + +def read_warc(warc_path): + records = {'warcinfo': [], 'response': [], 'metadata': []} + with fsspec.open(warc_path, 'rb') as stream: + for record in ArchiveIterator(stream): + content = record.content_stream().read().decode('utf-8', errors='ignore') + records.setdefault(record.rec_type, []).append((record, content)) + return records + + +def assert_repackage( + tmpdir, + warc_download_prefix: str, + base_prefix, + extra_args: Optional[List[str]] = None, + warc_filename: str = 'TEST_warc_by_index-001.warc.gz', + s3_env=None, +): + base_prefix = str(base_prefix) + main( + args=[ + '-v', + 'repackage', + write_captures(tmpdir, s3_env=s3_env if warc_download_prefix.startswith('s3://') else None), + '--no-sort', # keep the order of the CDX file + '--add-metadata-record', + str(METADATA_RECORD_PATH), + f'--prefix={base_prefix}/TEST_warc_by_index', + '--creator=foo', + '--operator=bob', + f'--warc-download-prefix={warc_download_prefix}', + ] + + (extra_args or []) + ) + + # with --processes, each worker writes its own WARC files: read the whole series + fs, fs_prefix = fsspec.url_to_fs(f'{base_prefix}/TEST_warc_by_index') + warc_paths = [fs.unstrip_protocol(p) for p in sorted(fs.glob(fs_prefix + '-*.warc.gz'))] + assert warc_paths[0].endswith('/' + warc_filename) + records = {'warcinfo': [], 'response': [], 'metadata': []} + for warc_path in warc_paths: + file_records = read_warc(warc_path) + # each file is a complete WARC whose warcinfo record names the file itself + (info_record, _), = file_records['warcinfo'] + assert info_record.rec_headers.get('WARC-Filename') == warc_path.rsplit('/', 1)[-1] + for rec_type, recs in file_records.items(): + records[rec_type] += recs + + assert len(records['response']) == 10, 'Invalid record count' + + assert len(records['warcinfo']) == len(warc_paths) + info_record, info_content = records['warcinfo'][0] + assert info_record.rec_headers.get('WARC-Filename') == warc_filename + assert 'operator: bob' in info_content, 'Invalid info record' + + response_contents = [content for _, content in records['response']] + if any(arg.startswith('--processes') for arg in extra_args or []): + # the captures are sharded across processes + assert any('Catalogue en ligne Mission de France' in c for c in response_contents) + else: + assert 'Catalogue en ligne Mission de France' in response_contents[0], 'Invalid response content' + assert 'dojo/dijit/themes/tundra/tundra' in response_contents[9], 'Invalid response content' + + # one metadata record per WARC file + assert len(records['metadata']) == len(warc_paths) + metadata_record, metadata_content = records['metadata'][0] + assert metadata_content[:10] == 'example.co', 'Invalid metadata record' + assert metadata_record.rec_headers.get('WARC-Payload-Digest') == 'sha1:VXA2A5YUS3TAY36AUO6MACRMNOH5RXG2' + with open(METADATA_RECORD_PATH, 'rb') as f: + assert metadata_record.length == len(f.read()) + + +def test_repackage_over_http(tmpdir): + assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=tmpdir) + + +def test_repackage_over_http_with_parallel_readers(tmpdir): + assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=tmpdir, extra_args=['--parallel-readers=3']) + + +def test_repackage_over_http_with_processes(tmpdir, monkeypatch, mock_data_commoncrawl): + if mock_data_commoncrawl is not None: + # the HTTP mock only applies in this process + monkeypatch.setattr('cdx_toolkit.repackage.multiprocess.process_pool', InlineExecutor) + assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=tmpdir, extra_args=['--processes=2']) + + +@pytest.mark.parametrize('fmt', ['csv', 'tsv', 'jsonl']) +def test_repackage_input_formats(tmpdir, fmt): + """Captures written by `cdxt index` in any format (here: with the columnar index column names for csv).""" + captures = list(read_records(write_captures(tmpdir))) + fields = ['url', 'filename', 'offset', 'length'] + if fmt == 'csv': + for c in captures: + c['warc_filename'], c['warc_record_offset'], c['warc_record_length'] = ( + c.pop('filename'), c.pop('offset'), c.pop('length') + ) + fields = ['url', 'warc_filename', 'warc_record_offset', 'warc_record_length'] + path = os.path.join(str(tmpdir), f'captures.{fmt}') + write_records(captures, path, fmt, fields) + + main(args=['repackage', path, f'--prefix={tmpdir}/TEST']) + main(args=['repackage', path, f'--input-format={fmt}', f'--prefix={tmpdir}/TEST2']) + + for prefix in ('TEST', 'TEST2'): + responses = read_warc(os.path.join(str(tmpdir), f'{prefix}-001.warc.gz'))['response'] + assert len(responses) == 10 + + +def test_repackage_subprefix_and_metadata(tmpdir): + main( + args=[ + 'repackage', + write_captures(tmpdir, n=1), + f'--prefix={tmpdir}/TEST', + '--subprefix=SUB', + '--creator=test_creator', + '--operator=test_operator', + '--is-part-of=PART', + '--description=DESC', + ] + ) + + (_, info), = read_warc(os.path.join(str(tmpdir), 'TEST-SUB-001.warc.gz'))['warcinfo'] + assert 'creator: test_creator' in info + assert 'operator: test_operator' in info + assert 'isPartOf: PART' in info + assert 'description: DESC' in info + + +def test_repackage_without_creator_operator(tmpdir): + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={tmpdir}/TEST_NO_META']) + + (_, info), = read_warc(os.path.join(str(tmpdir), 'TEST_NO_META-001.warc.gz'))['warcinfo'] + assert 'creator:' not in info + assert 'operator:' not in info + + +def test_repackage_glob_input(tmpdir): + write_captures(tmpdir, n=1, name='a.cdxj') + write_captures(tmpdir, n=2, name='b.cdxj') + + main(args=['repackage', f'{tmpdir}/*.cdxj', f'--prefix={tmpdir}/TEST']) + + assert len(read_warc(os.path.join(str(tmpdir), 'TEST-001.warc.gz'))['response']) == 3 + + +def test_repackage_no_input_files_found(tmpdir, capsys): + with pytest.raises(SystemExit) as exc_info: + main(args=['repackage', f'{tmpdir}/nonexistent-*.gz', f'--prefix={tmpdir}/TEST']) + + assert exc_info.value.code == 2 + assert 'no files found' in capsys.readouterr().err + + +@pytest.mark.parametrize('option', ['--processes', '--parallel-readers']) +@pytest.mark.parametrize('value', ['0', '-1', 'x']) +def test_repackage_rejects_non_positive_counts(tmpdir, capsys, option, value): + with pytest.raises(SystemExit) as exc_info: + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={tmpdir}/TEST', option, value]) + assert exc_info.value.code == 2 + assert 'positive integer' in capsys.readouterr().err + + +def test_repackage_rejects_an_existing_series(tmpdir, capsys): + """A second run with the same prefix would overwrite the first one's files.""" + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={tmpdir}/TEST']) + first = read_warc(os.path.join(str(tmpdir), 'TEST-001.warc.gz'))['response'] + + with pytest.raises(SystemExit) as exc_info: + main(args=['repackage', write_captures(tmpdir, n=2), f'--prefix={tmpdir}/TEST']) + + assert exc_info.value.code == 2 + assert 'exist already' in capsys.readouterr().err + assert read_warc(os.path.join(str(tmpdir), 'TEST-001.warc.gz'))['response'][0][1] == first[0][1] + assert not os.path.exists(os.path.join(str(tmpdir), 'TEST-002.warc.gz')) + # another subprefix is another series + main(args=['repackage', write_captures(tmpdir, n=2), f'--prefix={tmpdir}/TEST', '--subprefix=B']) + assert len(read_warc(os.path.join(str(tmpdir), 'TEST-B-001.warc.gz'))['response']) == 2 + + +@requires_aws_s3 +def test_repackage_rejects_an_existing_series_on_s3(tmpdir, s3_tmpdir): + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={s3_tmpdir}/TEST']) + with pytest.raises(SystemExit) as exc_info: + main(args=['repackage', write_captures(tmpdir, n=1), f'--prefix={s3_tmpdir}/TEST']) + assert exc_info.value.code == 2 + + +@requires_aws_s3 +def test_repackage_over_s3(tmpdir, s3_env): + assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir, s3_env=s3_env) + + +@requires_aws_s3 +def test_repackage_over_s3_to_s3(tmpdir, s3_tmpdir, s3_env): + assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=s3_tmpdir, s3_env=s3_env) + + +@requires_aws_s3 +def test_repackage_over_s3_to_s3_with_parallel_readers(tmpdir, s3_tmpdir, s3_env): + assert_repackage( + tmpdir, 's3://commoncrawl', base_prefix=s3_tmpdir, extra_args=['--parallel-readers=3', '--is-part-of=foobar'], + s3_env=s3_env, + ) + + +@requires_aws_s3 +def test_repackage_over_http_to_s3_with_parallel_readers(tmpdir, s3_tmpdir): + assert_repackage(tmpdir, 'https://data.commoncrawl.org', base_prefix=s3_tmpdir, extra_args=['--parallel-readers=3']) + + +@requires_aws_s3 +def test_repackage_over_s3_to_local_with_processes(tmpdir, s3_env, monkeypatch): + # the worker processes use the mocked S3 too: it is configured with environment variables, which a spawned + # process gets from this process (a forkserver, the default since Python 3.14 on Linux, could have been + # started before, with other environment variables, e.g. real AWS credentials) + spawn = multiprocessing.get_context('spawn') + monkeypatch.setattr('cdx_toolkit.repackage.multiprocess.multiprocessing.get_context', lambda *args: spawn) + assert_repackage(tmpdir, 's3://commoncrawl', base_prefix=tmpdir, extra_args=['--processes=2'], s3_env=s3_env) diff --git a/tests/repackage/test_repackage_function.py b/tests/repackage/test_repackage_function.py new file mode 100644 index 0000000..878087d --- /dev/null +++ b/tests/repackage/test_repackage_function.py @@ -0,0 +1,73 @@ +"""repackage(), the Python equivalent of `cdxt repackage`, with open_index() and IndexQuery for `cdxt index`.""" +import json +from unittest.mock import patch + +import pytest +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index +from cdx_toolkit.repackage import repackage +from cdx_toolkit.repackage.jobs import RepackageResult +from tests.fake_warc_server import FakeWarcServer + + +SERVER = FakeWarcServer('https://data.example.org') +CAPTURES = SERVER.add_captures('crawl-data/0.warc.gz', [f'https://example.com/{i}' for i in range(3)]) + + +def warc_records(path): + with open(path, 'rb') as f: + return [(r.rec_type, r.rec_headers.get_header('WARC-Target-URI')) for r in ArchiveIterator(f)] + + +def fetch_from_server(): + return patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=SERVER.get) + + +def test_repackage_records(tmp_path): + with fetch_from_server(): + result = repackage(CAPTURES, str(tmp_path / 'out'), warc_download_prefix='https://data.example.org') + + assert result == RepackageResult(requested=3, written=3) + assert warc_records(tmp_path / 'out-001.warc.gz') == [('warcinfo', None)] + [ + ('resource', c['url']) for c in CAPTURES] + + +def test_repackage_index_file(tmp_path): + path = tmp_path / 'captures.jsonl' + path.write_text(''.join(json.dumps(c) + '\n' for c in CAPTURES)) + with fetch_from_server(): + result = repackage(str(path), str(tmp_path / 'out'), warc_download_prefix='https://data.example.org', + subprefix='SUB', creator='me') + + assert result.written == 3 + with open(tmp_path / 'out-SUB-001.warc.gz', 'rb') as f: + warcinfo = next(iter(ArchiveIterator(f))) + fields = warcinfo.content_stream().read().decode() + assert 'isPartOf: ' + str(tmp_path / 'out') + '-SUB' in fields + assert 'creator: me' in fields + assert 'description: warc extraction generated with cdx_toolkit' in fields + + +def test_open_index_and_repackage(tmp_path): + """The two steps in Python, as `cdxt index INDEX --domain example.com | cdxt repackage`.""" + index = tmp_path / 'index.cdx' + index.write_text(''.join( + f'com,example)/{c["url"].rsplit("/", 1)[1]} 20240101000000 {json.dumps(c)}\n' for c in CAPTURES)) + + captures = open_index(str(index)).records(IndexQuery(domains='example.com', limit=2)) + with fetch_from_server(): + result = repackage(captures, str(tmp_path / 'out'), warc_download_prefix='https://data.example.org') + + assert result == RepackageResult(requested=2, written=2) + assert [url for _, url in warc_records(tmp_path / 'out-001.warc.gz')[1:]] == [c['url'] for c in CAPTURES[:2]] + + +def test_repackage_usage_errors(tmp_path): + (tmp_path / 'out-001.warc.gz').write_bytes(b'earlier run') + with pytest.raises(UsageError, match='exist already'): + repackage(CAPTURES, str(tmp_path / 'out')) + with pytest.raises(UsageError, match='--processes cannot be used with --wb'): + repackage(CAPTURES, str(tmp_path / 'new'), wb='ia', processes=2) + with pytest.raises(UsageError, match='no WARC location'): + repackage([{'url': 'https://example.com/'}], str(tmp_path / 'new')) diff --git a/tests/repackage/test_repackager.py b/tests/repackage/test_repackager.py new file mode 100644 index 0000000..dbc7f84 --- /dev/null +++ b/tests/repackage/test_repackager.py @@ -0,0 +1,725 @@ +"""The Repackager pipeline end to end without network: fake HTTP and HF reads, in-process "processes".""" +import asyncio +import contextlib +import glob +import io +import gzip +import itertools +import logging +import os +import sys +import threading +import time +from unittest.mock import Mock, patch + +import fsspec +import pytest +import requests +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit import myrequests +from cdx_toolkit.repackage import multiprocess +from cdx_toolkit.repackage.jobs import RangeJob, RepackageResult +from cdx_toolkit.repackage.readers.http import HttpRangeReader, RangeError +from cdx_toolkit.repackage.readers.s3 import S3RangeReader +from cdx_toolkit.repackage.writers.local import LocalOutputFile +from cdx_toolkit.repackage.jobs import RecordsSource, WaybackSource +from cdx_toolkit.repackage.repackager import Repackager, ThroughputTracker +from tests.conftest import InlineExecutor +from tests.fake_warc_server import FakeWarcServer +from tests.conftest import requires_aioboto3 + + +def make_server(): + """Four captures in two WARC files.""" + server = FakeWarcServer('https://data.example.org') + captures = [] + for i in range(4): + captures += server.add_captures(f'crawl-data/{i % 2}.warc.gz', [f'https://example.com/{i}']) + return server, captures + + +SERVER, CAPTURES = make_server() +fake_get = SERVER.get + + +def response_records(paths): + records = [] + for path in sorted(paths): + with fsspec.open(path, 'rb') as f: + records += [r for r in ArchiveIterator(f) if r.rec_type == 'resource'] + return records + + +def test_failed_captures_are_counted_and_the_others_written(tmp_path): + calls = [] + + def flaky_get(url, headers=None, **kwargs): + calls.append(url) + if len(calls) == 1: + raise ConnectionError('network down') + return fake_get(url, headers) + + repackager = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + ) + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=flaky_get): + assert repackager.run() == RepackageResult(requested=4, written=3, failed=1) + assert len(response_records(glob.glob(str(tmp_path / 'out-*.warc.gz')))) == 3 + + +@pytest.mark.parametrize('response', [ + Mock(status_code=200, headers={}, content=b'x' * 100), # range ignored + Mock(status_code=301, headers={'Location': 'https://elsewhere.example.org/'}, content=b''), # redirect + Mock(status_code=206, headers={'Content-Range': 'bytes 0-0/100'}, content=b'x'), # truncated / wrong range + 'wrong-range', +]) +def test_invalid_range_responses_are_not_written(tmp_path, response): + capture = CAPTURES[1] # offset > 0 + offset, length = int(capture['offset']), int(capture['length']) + if response == 'wrong-range': + response = Mock(status_code=206, headers={'Content-Range': f'bytes {offset + 1}-{offset + length}/9999'}, + content=b'x' * length) + repackager = Repackager( + source=RecordsSource([capture], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + ) + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=response): + assert repackager.run() == RepackageResult(requested=1, written=0, failed=1) + assert glob.glob(str(tmp_path / 'out-*')) == [] + + +def test_a_slow_http_read_does_not_block_the_other_readers_and_the_writer(tmp_path): + """The first read is held until the second capture has been written.""" + second_written = threading.Event() + original_write = LocalOutputFile.write + + def get(url, headers=None, **kwargs): + if url.endswith(CAPTURES[0]['filename']): + assert second_written.wait(timeout=5), 'the second capture was not written while the first was read' + return fake_get(url, headers) + + async def write(self, data): + await original_write(self, data) + if b'https://example.com/1' in gzip.decompress(data): + second_written.set() + + repackager = Repackager( + source=RecordsSource(CAPTURES[:2], 'https://data.example.org', sort=False), prefix_path=str(tmp_path / 'out'), + writer_info={}, n_parallel_readers=2, + ) + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=get), \ + patch.object(LocalOutputFile, 'write', write): + assert repackager.run() == RepackageResult(requested=2, written=2) + records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) + assert [r.rec_headers.get_header('WARC-Target-URI') for r in records] == [CAPTURES[1]['url'], CAPTURES[0]['url']] + + +def test_writer_error_stops_the_pipeline_and_removes_the_file(tmp_path): + repackager = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + max_file_size=1, + ) + writes = [] + + original_write = LocalOutputFile.write + + async def failing_write(self, data): + writes.append(data) + if len(writes) == 4: # warcinfo + record of the first file, warcinfo + record of the second + raise OSError('disk full') + await original_write(self, data) + + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ + patch.object(LocalOutputFile, 'write', failing_write), pytest.raises(OSError, match='disk full'): + repackager.run() + # the first file is complete, the second was removed + assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == ['out-001.warc.gz'] + + +@pytest.mark.parametrize('failing_close, expected_files', [ + (1, []), # rotation: the file being closed is removed, no next file is started + (2, ['out-001.warc.gz']), # the final close: the last file is removed, the complete first one stays +]) +def test_close_error_stops_the_pipeline_and_removes_the_file(tmp_path, failing_close, expected_files): + repackager = Repackager( + source=RecordsSource(CAPTURES[:2], 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}, max_file_size=1, + ) + flushes = [] + original_flush = LocalOutputFile._flush + + async def failing_flush(self): + flushes.append(self.file_path) + if len(flushes) == failing_close: + raise OSError('disk full') + await original_flush(self) + + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ + patch.object(LocalOutputFile, '_flush', failing_flush), pytest.raises(OSError, match='disk full'): + repackager.run() + assert [os.path.basename(p) for p in glob.glob(str(tmp_path / 'out-*'))] == expected_files + + +def test_rotation_by_size(tmp_path): + repackager = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + max_file_size=1, + ) + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): + assert repackager.run().written == 4 + paths = sorted(glob.glob(str(tmp_path / 'out-*.warc.gz'))) + assert [os.path.basename(p) for p in paths] == [f'out-00{i}.warc.gz' for i in range(1, 5)] + assert all(len(response_records([p])) == 1 for p in paths) + + +def test_cancellation_with_a_full_queue_and_a_blocked_reader(tmp_path): + """Cancelling must not wait for queue space: the producer thread stops and the current file is removed.""" + source_done = threading.Event() + + class EndlessSource: + def iter_range_jobs(self): + try: + for i in itertools.count(): + yield RangeJob(url=f'https://data.example.org/{i}', offset=0, length=1) + finally: + source_done.set() + + reads = [] + + async def blocking_read(self, job): + reads.append(job.url) + if len(reads) > 1: + await asyncio.Event().wait() # blocks forever + return b'x' + + repackager = Repackager( + source=EndlessSource(), prefix_path=str(tmp_path / 'out'), writer_info={}, range_jobs_queue_size=2, + ) + + async def run(): + task = asyncio.create_task(repackager.run_async()) + while len(reads) < 2: + await asyncio.sleep(0.01) + await asyncio.sleep(0.2) # the queue fills up, the producer waits + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=2) + + def main(): + with patch.object(HttpRangeReader, 'read', blocking_read): + asyncio.run(run()) + + thread = threading.Thread(target=main, daemon=True) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive(), 'shutdown did not finish' + assert source_done.wait(timeout=1), 'the producer thread is still waiting' + assert glob.glob(str(tmp_path / 'out-*')) == [] + + +def cancel_http_read_after_the_first_request(job, request_started): + """Cancel the read once the first request has started; returns how long asyncio.run() took after cancelling.""" + async def main(): + task = asyncio.create_task(HttpRangeReader().read(job)) + while not request_started.is_set(): + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + return time.monotonic() + + cancelled_at = asyncio.run(main()) + return time.monotonic() - cancelled_at + + +def test_cancelled_http_read_does_not_wait_for_the_request(monkeypatch): + """asyncio.run() returns while the (uninterruptible) request is still blocked.""" + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'blocked.example.org', 0) + started, release = threading.Event(), threading.Event() + + def blocking_get(*args, **kwargs): + started.set() + release.wait(timeout=10) + raise requests.exceptions.ConnectionError('released') + + try: + with patch('cdx_toolkit.myrequests.requests.get', side_effect=blocking_get): + job = RangeJob(url='https://blocked.example.org/a.warc.gz', offset=0, length=1) + assert cancel_http_read_after_the_first_request(job, started) < 1 + assert not release.is_set() + finally: + release.set() + + +def test_cancelled_http_read_stops_retrying(monkeypatch): + """The thread does not keep retrying (here: waiting 60s after a connection error) after the cancellation.""" + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'down.example.org', 0) + started, finished = threading.Event(), threading.Event() + requests_made = [] + + def failing_get(*args, **kwargs): + requests_made.append(args) + started.set() + raise requests.exceptions.ConnectionError('down') + + def get(*args, **kwargs): + try: + return myrequests.myrequests_get(*args, **kwargs) + finally: + finished.set() + + with patch('cdx_toolkit.myrequests.requests.get', side_effect=failing_get), \ + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=get): + job = RangeJob(url='https://down.example.org/a.warc.gz', offset=0, length=1) + cancel_http_read_after_the_first_request(job, started) + assert finished.wait(timeout=2), 'the request thread is still waiting to retry' + assert len(requests_made) == 1 + + +def test_hf_reads_use_the_hf_reader(tmp_path): + repackager = Repackager( + source=RecordsSource(CAPTURES, 'hf://buckets/commoncrawl/commoncrawl'), prefix_path=str(tmp_path / 'out'), + writer_info={}, + ) + with patch('cdx_toolkit.repackage.readers.hf.make_hf_reader', fake_make_hf_reader): + assert repackager.run().written == 4 + records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) + assert sorted(r.rec_headers.get_header('WARC-Target-URI') for r in records) == sorted(c['url'] for c in CAPTURES) + + +@pytest.mark.parametrize('uvloop_installed', [True, False]) +def test_uvloop(monkeypatch, caplog, uvloop_installed): + monkeypatch.setenv('CDXT_UVLOOP', '1') + fake_uvloop = Mock(run=asyncio.run) if uvloop_installed else None + monkeypatch.setitem(__import__('sys').modules, 'uvloop', fake_uvloop) + repackager = Repackager(source=RecordsSource([], None), prefix_path='/nonexistent/out', writer_info={}) + + with caplog.at_level('INFO'): + assert repackager.run() == RepackageResult() + assert ('Using uvloop event loop' if uvloop_installed else 'uvloop is not installed') in caplog.text + + +class FakeS3Client: + """aioboto3 S3 client stand-in that serves the files of SERVER and stores multipart uploads.""" + + def __init__(self): + self.parts = {} + self.objects = {} + + async def create_multipart_upload(self, Bucket, Key): + return {'UploadId': Key} + + async def upload_part(self, Key, PartNumber, Body, **kwargs): + self.parts[(Key, PartNumber)] = Body + return {'ETag': str(PartNumber)} + + async def complete_multipart_upload(self, Bucket, Key, MultipartUpload, **kwargs): + parts = [self.parts[(Key, part['PartNumber'])] for part in MultipartUpload['Parts']] + self.objects[f's3://{Bucket}/{Key}'] = b''.join(parts) + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get_object(self, Bucket, Key, Range): + start, end = (int(n) for n in Range[len('bytes='):].split('-')) + + class Body: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def read(self): + return SERVER.files[Key][start:end + 1] + + return {'Body': Body()} + + +class FakeHfReader: + async def read(self, job): + return SERVER.files[job.url.split('/', 5)[-1]][job.offset:job.offset + job.length] + + +@contextlib.asynccontextmanager +async def fake_make_hf_reader(*args): + yield FakeHfReader() + + +def mixed_scheme_captures(): + """The captures with full s3:// and hf:// filenames, and a relative one (default download prefix).""" + s3, hf, relative = CAPTURES[0], CAPTURES[1], CAPTURES[2] + return [dict(s3, filename='s3://commoncrawl/' + s3['filename']), + dict(hf, filename='hf://buckets/commoncrawl/commoncrawl/' + hf['filename']), relative] + + +@requires_aioboto3 +@pytest.mark.parametrize('processes', [1, 2]) +def test_full_s3_and_hf_filenames_get_their_readers(tmp_path, processes): + source = RecordsSource(mixed_scheme_captures(), 'https://data.example.org') + with patch('aioboto3.Session') as session, \ + patch('cdx_toolkit.repackage.readers.hf.make_hf_reader', fake_make_hf_reader), \ + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ + patch.object(multiprocess, 'process_pool', InlineExecutor): + session.return_value.client.return_value = FakeS3Client() + if processes == 1: + result = Repackager(source=source, prefix_path=str(tmp_path / 'out'), writer_info={}).run() + else: + result = multiprocess.run_multiprocess_repackage( + source=source, n_processes=processes, readers_per_process=1, prefix_path=str(tmp_path / 'out'), + writer_info={}, writer_subprefix=None, write_paths_as_metadata_records=None, log_every_n=0, + max_attempts=1, uvloop=False, + ) + assert result == RepackageResult(requested=3, written=3) + records = response_records(glob.glob(str(tmp_path / 'out-*.warc.gz'))) + assert sorted(r.rec_headers.get_header('WARC-Target-URI') for r in records) == \ + sorted(c['url'] for c in CAPTURES[:3]) + + +@requires_aioboto3 +def test_s3_output_with_metadata_records(tmp_path): + metadata = tmp_path / 'urls.txt' + metadata.write_bytes(b'https://example.com/\n') + s3 = FakeS3Client() + with patch('aioboto3.Session') as session, \ + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): + session.return_value.client.return_value = s3 + result = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path='s3://bucket/out', writer_info={}, + write_paths_as_metadata_records=[str(metadata)], + ).run() + assert result.written == 4 + data = s3.objects['s3://bucket/out-001.warc.gz'] + records = [(r.rec_type, r.content_stream().read()) for r in ArchiveIterator(io.BytesIO(data))] + assert [rec_type for rec_type, _ in records] == ['warcinfo', 'metadata'] + ['resource'] * 4 + assert records[1][1] == b'https://example.com/\n' + + +def test_failing_metadata_record_removes_the_file(tmp_path): + repackager = Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), writer_info={}, + write_paths_as_metadata_records=[str(tmp_path / 'missing.txt')], + ) + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ + pytest.raises(FileNotFoundError): + repackager.run() + assert glob.glob(str(tmp_path / 'out-*')) == [] + + +class FakeS3Responses: + """An S3 client (client()) whose responses have the given bodies (bytes, or an exception raised when reading).""" + + def __init__(self, *bodies): + self.bodies = list(bodies) + self.requests = 0 + self.closed = 0 + + def client(self): + readers = self + + class Body: + def __init__(self, content): + self.content = content + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + readers.closed += 1 + + async def read(self): + if isinstance(self.content, Exception): + raise self.content + return self.content + + class Client: + async def get_object(self, **kwargs): + readers.requests += 1 + return {'Body': Body(readers.bodies.pop(0))} + + return Client() + + +def test_s3_reads_check_the_length(): + job = RangeJob(url='s3://commoncrawl/crawl-data/a.warc.gz', offset=0, length=10) + with pytest.raises(RangeError, match='requested 10 bytes'): + asyncio.run(S3RangeReader(FakeS3Responses(b'short').client(), 1, 0).read(job)) + + +@requires_aioboto3 +def test_s3_reads_retry_the_whole_transfer(): + from botocore.exceptions import ReadTimeoutError + + responses = FakeS3Responses( + ReadTimeoutError(endpoint_url='s3://x'), ConnectionResetError(), b'short', b'0123456789' + ) + job = RangeJob(url='s3://commoncrawl/crawl-data/a.warc.gz', offset=0, length=10) + assert asyncio.run(S3RangeReader(responses.client(), 5, 0).read(job)) == b'0123456789' + assert responses.requests == 4 + assert responses.closed == 4 # also the failed responses + + +@requires_aioboto3 +def test_no_s3_client_without_s3(tmp_path): + with patch('aioboto3.Session') as session, \ + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): + Repackager( + source=RecordsSource(CAPTURES, 'https://data.example.org'), prefix_path=str(tmp_path / 'out'), + writer_info={}, + ).run() + session.assert_not_called() + + +@pytest.mark.parametrize('remote', [False, True]) +def test_multiprocess_repackage(tmp_path, remote): + prefix = 'memory://mp-test/out/RUN' if remote else str(tmp_path / 'RUN') + with patch.object(multiprocess, 'process_pool', InlineExecutor), \ + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): + n = multiprocess.run_multiprocess_repackage( + source=RecordsSource(CAPTURES, 'https://data.example.org'), n_processes=3, readers_per_process=2, + prefix_path=prefix, writer_info={'isPartOf': 'test'}, writer_subprefix='SUB', + write_paths_as_metadata_records=None, log_every_n=1, aws_region_name='us-east-1', max_attempts=1, + uvloop=False, max_file_size=1, + ) + + assert n == RepackageResult(requested=4, written=4) + fs = fsspec.filesystem('memory') + if remote: + paths = ['memory://' + p for p in fs.glob('/mp-test/out/RUN-SUB-*.warc.gz')] + else: + paths = glob.glob(str(tmp_path / 'RUN-SUB-*.warc.gz')) + try: + # one file per record (--size 1), numbered by the shared counter + assert sorted(os.path.basename(p) for p in paths) == [f'RUN-SUB-00{i}.warc.gz' for i in range(1, 5)] + assert len(response_records(paths)) == 4 + for path in paths: # the warcinfo record names the file it is in + with fsspec.open(path, 'rb') as f: + warcinfo = next(iter(ArchiveIterator(f))) + assert warcinfo.rec_headers.get_header('WARC-Filename') == os.path.basename(path) + finally: + if remote: + fs.rm('/mp-test', recursive=True) + + +def test_multiprocess_repackage_keeps_the_order_of_the_source(tmp_path): + """Workers do not sort their shard again (--no-sort streams, a sorted source is sorted already).""" + captures = CAPTURES[::-1] + with patch.object(multiprocess, 'process_pool', InlineExecutor), \ + patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): + multiprocess.run_multiprocess_repackage( + source=RecordsSource(captures, 'https://data.example.org', sort=False), n_processes=1, + readers_per_process=1, prefix_path=str(tmp_path / 'RUN'), writer_info={}, writer_subprefix=None, + write_paths_as_metadata_records=None, log_every_n=1, max_attempts=1, uvloop=False, + ) + records = response_records(glob.glob(str(tmp_path / 'RUN-*.warc.gz'))) + assert [r.rec_headers.get_header('WARC-Target-URI') for r in records] == [c['url'] for c in captures] + + +def test_multiprocess_repackage_of_wayback_replays(tmp_path): + """hf:// (and other non-S3 remote) output goes through the shards also for --wb: the shards hold the captures.""" + from warcio.recordbuilder import RecordBuilder + + def fake_fetch_wb_warc(capture, wb): + assert wb == 'https://web.archive.org/web' + return RecordBuilder().create_warc_record(capture['url'], 'resource', payload=io.BytesIO(b'x')) + + captures = [{'url': f'https://example.com/{i}', 'timestamp': '20240101000000', 'status': '200'} for i in range(3)] + fs = fsspec.filesystem('memory') + with patch.object(multiprocess, 'process_pool', InlineExecutor), \ + patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc', side_effect=fake_fetch_wb_warc): + result = multiprocess.run_multiprocess_repackage( + source=WaybackSource(captures, 'ia'), n_processes=1, readers_per_process=1, + prefix_path='memory://wb-test/RUN', writer_info={}, writer_subprefix=None, + write_paths_as_metadata_records=None, log_every_n=1, max_attempts=1, uvloop=False, + ) + try: + assert result == RepackageResult(requested=3, written=3) + with fsspec.open('memory://wb-test/RUN-001.warc.gz', 'rb') as f: + uris = [r.rec_headers.get_header('WARC-Target-URI') for r in ArchiveIterator(f) if r.rec_type == 'resource'] + assert uris == [c['url'] for c in captures] + finally: + fs.rm('/wb-test', recursive=True) + + +def test_at_least_one_reader(): + with pytest.raises(ValueError, match='n_parallel_readers'): + Repackager(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=0) + + +def test_log_progress(caplog): + """Progress is logged every --log-every-n items.""" + caplog.set_level(logging.INFO, logger='cdx_toolkit.repackage.repackager') + + repackager = Repackager(source=RecordsSource([], None), prefix_path='/fake/prefix', writer_info={}, log_every_n=2) + tracker = ThroughputTracker() + for _ in range(5): + tracker.add(bytes_count=1024 * 1024, records_count=1) + repackager.log_progress('WARC Reader 1', tracker) + + assert caplog.text.count('WARC Reader 1: ') == 2 + assert 'WARC Reader 1: 4 items, 4.0 MB in ' in caplog.text + + +def test_writer_error_with_a_blocked_source(tmp_path): + """The writer fails while the source waits for its next capture (e.g. a generator waiting for input): run() + raises the writer's error at once, without waiting for the source.""" + release = threading.Event() + + def captures(): + yield CAPTURES[0] + release.wait() # blocks until the end of the test + yield CAPTURES[1] + + async def failing_write(self, data): + raise OSError('disk full') + + outcome = [] + + def main(): + repackager = Repackager( + source=RecordsSource(captures(), 'https://data.example.org', sort=False), + prefix_path=str(tmp_path / 'out'), writer_info={}, + ) + try: + repackager.run() + except OSError as e: + outcome.append(e) + + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \ + patch.object(LocalOutputFile, 'write', failing_write): + thread = threading.Thread(target=main, daemon=True) + thread.start() + thread.join(timeout=5) + release.set() + assert not thread.is_alive(), 'run() waits for the blocked source' + assert [str(e) for e in outcome] == ['disk full'] + assert glob.glob(str(tmp_path / 'out-*')) == [] + + +CHILD = ''' +import faulthandler +import sys +from unittest.mock import patch +faulthandler.dump_traceback_later(20, exit=True) # the stacks of all threads if it hangs +from cdx_toolkit.cli import main +from cdx_toolkit.repackage.writers.local import LocalOutputFile +from tests.repackage.test_repackager import fake_get + +async def failing_write(self, data): + raise OSError('disk full') + +with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get), \\ + patch.object(LocalOutputFile, 'write', failing_write): + main(['repackage', '-', '--no-sort', '--warc-download-prefix', 'https://data.example.org', + '--prefix', sys.argv[1]]) +''' + + +def test_writer_error_with_stdin_still_open(tmp_path): + """cdxt repackage reading stdin from a process that keeps running: the writer's error ends cdxt at once.""" + import json + import subprocess + import sys + + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + # stderr to a file: a pipe that is not read while the process runs would block it once full (a few KB on + # Windows) + stderr_path = tmp_path / 'stderr.txt' + with open(stderr_path, 'w') as stderr_file: + proc = subprocess.Popen( + [sys.executable, '-c', CHILD, str(tmp_path / 'out')], cwd=root, stdin=subprocess.PIPE, + stderr=stderr_file, text=True, + ) + try: + proc.stdin.write(json.dumps(CAPTURES[0]) + '\n') + proc.stdin.flush() # and stdin stays open + proc.wait(timeout=30) + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + proc.stdin.close() + stderr = stderr_path.read_text() + assert proc.returncode != 0, stderr + assert 'disk full' in stderr, stderr + assert glob.glob(str(tmp_path / 'out-*')) == [] + + +CHILD_WITHOUT_FAILURE = ''' +import sys +from unittest.mock import patch +from cdx_toolkit.cli import main +from tests.repackage.test_repackager import fake_get + +with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): + main(['repackage', '-', '--no-sort', '--warc-download-prefix', 'https://data.example.org', + '--prefix', sys.argv[1]]) +''' + + +@pytest.mark.skipif(sys.platform == 'win32', reason='SIGINT') +def test_ctrl_c_with_stdin_still_open(tmp_path): + """Ctrl-C while cdxt repackage waits for more input on stdin: it ends at once, and removes the current file.""" + import json + import signal + import subprocess + + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + proc = subprocess.Popen( + [sys.executable, '-c', CHILD_WITHOUT_FAILURE, str(tmp_path / 'out')], cwd=root, stdin=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True, # not a pipe that nobody reads (it would block the process once full) + ) + try: + proc.stdin.write(json.dumps(CAPTURES[0]) + '\n') + proc.stdin.flush() # and stdin stays open + deadline = time.monotonic() + 30 + while not glob.glob(str(tmp_path / 'out-*')) and time.monotonic() < deadline: + time.sleep(0.05) # the first record is being written + proc.send_signal(signal.SIGINT) + proc.wait(timeout=30) + finally: + if proc.poll() is None: + proc.kill() + proc.stdin.close() + assert proc.returncode != 0 + assert glob.glob(str(tmp_path / 'out-*')) == [] + + +def test_cancel_event_stops_the_pipeline_and_removes_the_file(tmp_path): + """The event set by the parent of the worker processes (when another worker failed).""" + from cdx_toolkit.repackage.repackager import RepackageCancelled + + release, cancel = threading.Event(), threading.Event() + + def captures(): + yield CAPTURES[0] + release.wait() + yield CAPTURES[1] + + outcome = [] + + def main(): + repackager = Repackager( + source=RecordsSource(captures(), 'https://data.example.org', sort=False), + prefix_path=str(tmp_path / 'out'), writer_info={}, cancel=cancel, + ) + try: + outcome.append(repackager.run()) + except RepackageCancelled as e: + outcome.append(e) + + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=fake_get): + thread = threading.Thread(target=main, daemon=True) + thread.start() + deadline = time.monotonic() + 10 + while not glob.glob(str(tmp_path / 'out-*')) and time.monotonic() < deadline: + time.sleep(0.01) # the first record is being written + cancel.set() + thread.join(timeout=5) + release.set() + assert not thread.is_alive() + assert len(outcome) == 1 and isinstance(outcome[0], RepackageCancelled) + assert glob.glob(str(tmp_path / 'out-*')) == [] diff --git a/tests/repackage/test_retries.py b/tests/repackage/test_retries.py new file mode 100644 index 0000000..714cb1a --- /dev/null +++ b/tests/repackage/test_retries.py @@ -0,0 +1,177 @@ +import pytest + +import asyncio + + +from cdx_toolkit.repackage.retries import _backoff, with_retries +from cdx_toolkit.repackage.urls import parse_s3_uri + +# the retry tests raise botocore's exceptions (S3 support is optional: cdx_toolkit[s3]) +pytest.importorskip('botocore') +from botocore.exceptions import EndpointConnectionError # noqa: E402 +from botocore.exceptions import ClientError # noqa: E402 + + +def test_backoff(): + """Test _backoff function with exponential backoff and jitter.""" + base_backoff = 1.0 + tol_float = 0.001 # tolerance for float errors + + # Test attempt 1: should be between 0.8 and 1.2 seconds (with jitter) + result1 = _backoff(1, base_backoff) + assert 0.8 <= result1 <= 1.2 + tol_float + + # Test attempt 2: should be between 1.6 and 2.41 seconds (2^1 * base * jitter) + result2 = _backoff(2, base_backoff) + assert 1.6 <= result2 <= 2.4 + tol_float + + # Test attempt 3: should be between 3.2 and 4.8 seconds (2^2 * base * jitter) + result3 = _backoff(3, base_backoff) + assert 3.2 <= result3 <= 4.8 + tol_float + + # Test with different base backoff + base_backoff_small = 0.1 + result_small = _backoff(1, base_backoff_small) + assert 0.08 <= result_small <= 0.12 + tol_float + + # Test minimum backoff (should never be less than 0.05) + very_small_base = 0.001 + result_min = _backoff(1, very_small_base) + assert result_min >= 0.05 + + # Test that backoff increases with attempts + results = [_backoff(i, 0.5) for i in range(1, 6)] + # Generally should increase, though jitter might cause small variations + # Check that the trend is generally increasing + assert results[1] > results[0] * 0.8 # Allow for jitter variation + assert results[2] > results[1] * 0.8 + assert results[3] > results[2] * 0.8 + + +def test_parse_s3_uri(): + """Test parse_s3_uri function for valid and invalid S3 URIs.""" + + # Test valid S3 URIs + bucket, prefix = parse_s3_uri('s3://my-bucket/path/to/file.txt') + assert bucket == 'my-bucket' + assert prefix == 'path/to/file.txt' + + bucket, prefix = parse_s3_uri('s3://test-bucket/folder/subfolder/data.json') + assert bucket == 'test-bucket' + assert prefix == 'folder/subfolder/data.json' + + bucket, prefix = parse_s3_uri('s3://simple/file') + assert bucket == 'simple' + assert prefix == 'file' + + # Test with deep nested paths + bucket, prefix = parse_s3_uri('s3://bucket/a/b/c/d/e/f/file.ext') + assert bucket == 'bucket' + assert prefix == 'a/b/c/d/e/f/file.ext' + + # Test invalid URIs - should raise ValueError + with pytest.raises(ValueError, match='Not an S3 URI'): + parse_s3_uri('http://example.com/path') + + with pytest.raises(ValueError, match='Not an S3 URI'): + parse_s3_uri('ftp://bucket/file') + + with pytest.raises(ValueError, match='Not an S3 URI'): + parse_s3_uri('bucket/file') + + # Test malformed S3 URIs + with pytest.raises(ValueError, match='Malformed S3 URI'): + parse_s3_uri('s3://') + + with pytest.raises(ValueError, match='Malformed S3 URI'): + parse_s3_uri('s3://bucket') + + with pytest.raises(ValueError, match='Malformed S3 URI'): + parse_s3_uri('s3://bucket/') + + with pytest.raises(ValueError, match='Malformed S3 URI'): + parse_s3_uri('s3:///file') + + +def test_with_retries_success(): + """Test with_retries function with successful operation on first attempt.""" + + async def run_test(): + call_count = 0 + + async def successful_coro(): + nonlocal call_count + call_count += 1 + return 'success' + + result = await with_retries(successful_coro, op_name='test_op', max_attempts=3, base_backoff_seconds=0.1) + + assert result == 'success' + assert call_count == 1 + + asyncio.run(run_test()) + + +def test_with_retries_eventual_success(): + """Test with_retries function that succeeds after initial failures.""" + + async def run_test(): + call_count = 0 + + async def eventually_successful_coro(): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise ClientError({'Error': {'Code': 'Throttling'}}, 'test_op') + return 'success' + + result = await with_retries( + eventually_successful_coro, + op_name='test_op', + max_attempts=3, + base_backoff_seconds=0.01, # Very short for testing + ) + + assert result == 'success' + assert call_count == 3 + + asyncio.run(run_test()) + + +def test_with_retries_max_attempts_exceeded(): + """Test with_retries function when max attempts are exceeded.""" + + async def run_test(): + call_count = 0 + + async def failing_coro(): + nonlocal call_count + call_count += 1 + raise EndpointConnectionError(endpoint_url='test') + + with pytest.raises(EndpointConnectionError): + await with_retries(failing_coro, op_name='test_op', max_attempts=2, base_backoff_seconds=0.01) + + assert call_count == 2 + + asyncio.run(run_test()) + + +def test_with_retries_non_retryable_exception(): + """Test with_retries function with non-retryable exceptions.""" + + async def run_test(): + call_count = 0 + + async def failing_coro(): + nonlocal call_count + call_count += 1 + raise ValueError('Non-retryable error') + + with pytest.raises(ValueError): + await with_retries(failing_coro, op_name='test_op', max_attempts=3, base_backoff_seconds=0.01) + + # Should fail immediately without retries + assert call_count == 1 + + asyncio.run(run_test()) diff --git a/tests/repackage/test_s3_output.py b/tests/repackage/test_s3_output.py new file mode 100644 index 0000000..1ce570b --- /dev/null +++ b/tests/repackage/test_s3_output.py @@ -0,0 +1,147 @@ +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from cdx_toolkit.repackage.writers.s3 import S3OutputFile, mpu_abort + + +def test_mpu_abort_success(): + """Test mpu_abort function with successful abort.""" + + async def run_test(): + mock_s3 = AsyncMock() + bucket = 'test-bucket' + key = 'test-key' + upload_id = 'test-upload-id' + + await mpu_abort(mock_s3, bucket, key, upload_id) + + mock_s3.abort_multipart_upload.assert_called_once_with(Bucket=bucket, Key=key, UploadId=upload_id) + + asyncio.run(run_test()) + + +def test_mpu_abort_with_exception(): + """Test mpu_abort function when abort fails (should catch exception).""" + + async def run_test(): + mock_s3 = AsyncMock() + mock_s3.abort_multipart_upload.side_effect = Exception('S3 error') + + bucket = 'test-bucket' + key = 'test-key' + upload_id = 'test-upload-id' + + # Should not raise exception, should log it instead + await mpu_abort(mock_s3, bucket, key, upload_id) + + mock_s3.abort_multipart_upload.assert_called_once_with(Bucket=bucket, Key=key, UploadId=upload_id) + + asyncio.run(run_test()) + + +class FakeS3: + """upload_part fails `fail_times` times (ClientError), then stores the part bodies.""" + + def __init__(self, fail_times=0, fail_complete=False): + self.fail_times = fail_times + self.fail_complete = fail_complete + self.bodies = {} + self.completed = None + self.aborted = False + + async def create_multipart_upload(self, Bucket, Key): + return {'UploadId': 'upload-id'} + + async def upload_part(self, PartNumber, Body, **kwargs): + if self.fail_times: + self.fail_times -= 1 + from botocore.exceptions import ClientError + + raise ClientError({'Error': {'Code': '500', 'Message': 'boom'}}, 'UploadPart') + self.bodies[PartNumber] = Body + return {'ETag': f'etag-{PartNumber}'} + + async def complete_multipart_upload(self, MultipartUpload, **kwargs): + if self.fail_complete: + raise RuntimeError('complete failed') + self.completed = b''.join(self.bodies[p['PartNumber']] for p in MultipartUpload['Parts']) + + async def abort_multipart_upload(self, **kwargs): + self.aborted = True + + +def make_writer(s3): + writer = S3OutputFile(s3, 'bucket', 'out.warc.gz', min_part_size=4, max_attempts=2, base_backoff_seconds=0) + writer.upload_id = 'upload-id' + return writer + + +def test_shard_writer_failed_part_aborts_the_upload(): + pytest.importorskip('botocore') + s3 = FakeS3(fail_times=2) # more than max_attempts + writer = make_writer(s3) + + async def run(): + with pytest.raises(Exception, match='boom'): + await writer.write(b'ABCD') + with pytest.raises(RuntimeError): + await writer.write(b'EFGH') + with pytest.raises(RuntimeError, match='aborted'): + await writer.close() + + asyncio.run(run()) + assert s3.aborted + assert s3.completed is None + + +def test_shard_writer_retried_part_keeps_every_byte_once(): + pytest.importorskip('botocore') + s3 = FakeS3(fail_times=1) # less than max_attempts + writer = make_writer(s3) + + async def run(): + await writer.write(b'ABCDEF') + await writer.write(b'GHIJ') + await writer.close() + + asyncio.run(run()) + assert s3.completed == b'ABCDEFGHIJ' + assert not s3.aborted + + +def test_shard_writer_uploads_parts_of_the_minimum_size(): + s3 = FakeS3() + writer = make_writer(s3) + + async def run(): + await writer.start() + await writer.write(b'AB') + assert s3.bodies == {} # buffered + await writer.write(b'CDEFGHIJK') + assert s3.bodies == {1: b'ABCD', 2: b'EFGH'} + await writer.close() + + asyncio.run(run()) + assert s3.bodies[3] == b'IJK' + assert s3.completed == b'ABCDEFGHIJK' + + +def test_shard_writer_without_data_completes_nothing(): + s3 = FakeS3() + asyncio.run(make_writer(s3).close()) + assert s3.completed is None and not s3.aborted + + +def test_shard_writer_failed_complete_aborts_the_upload(): + s3 = FakeS3(fail_complete=True) + writer = make_writer(s3) + + async def run(): + await writer.write(b'AB') + with pytest.raises(RuntimeError, match='complete failed'): + await writer.close() + + asyncio.run(run()) + assert s3.aborted diff --git a/tests/repackage/test_series.py b/tests/repackage/test_series.py new file mode 100644 index 0000000..ae4bb97 --- /dev/null +++ b/tests/repackage/test_series.py @@ -0,0 +1,99 @@ +"""The series of output WARC files: numbering, rotation, warcinfo records, and aborting on failure.""" +import asyncio +import glob +import os +from io import BytesIO +from unittest.mock import patch + +import pytest +from warcio.archiveiterator import ArchiveIterator +from warcio.recordbuilder import RecordBuilder + +from cdx_toolkit.repackage.warc_records import get_bytes_from_warc_record +from cdx_toolkit.repackage.writers.local import LocalOutputFile +from cdx_toolkit.repackage.writers.series import WarcFileSeries + + +def record(url: str) -> bytes: + rec = RecordBuilder().create_warc_record(url, 'resource', payload=BytesIO(b'x' * 100)) + return get_bytes_from_warc_record(rec, gzip=True) + + +def records_by_file(prefix): + result = {} + for path in sorted(glob.glob(prefix + '-*.warc.gz')): + with open(path, 'rb') as f: + result[os.path.basename(path)] = [ + (r.rec_type, r.rec_headers.get_header('WARC-Filename') or r.rec_headers.get_header('WARC-Target-URI')) + for r in ArchiveIterator(f) + ] + return result + + +def write_all(series, records): + async def main(): + for data in records: + await series.write(data) + await series.close() + + asyncio.run(main()) + + +def test_no_file_without_records(tmp_path): + write_all(WarcFileSeries(str(tmp_path / 'out'), {}), []) + assert os.listdir(tmp_path) == [] + + +def test_rotation_and_warcinfo_per_file(tmp_path): + prefix = str(tmp_path / 'out') + data = [record(f'https://example.com/{i}') for i in range(3)] + # room for the warcinfo record and one record, not two + series = WarcFileSeries(prefix, {'software': 'test'}, writer_subprefix='SUB', max_file_size=len(data[0]) * 2) + write_all(series, data) + + assert records_by_file(prefix) == { + f'out-SUB-00{i + 1}.warc.gz': [('warcinfo', f'out-SUB-00{i + 1}.warc.gz'), + ('resource', f'https://example.com/{i}')] + for i in range(3) + } + assert series.sequence == 3 + + +def test_record_larger_than_the_size_limit_gets_a_file(tmp_path): + prefix = str(tmp_path / 'out') + write_all(WarcFileSeries(prefix, {}, max_file_size=1), [record('https://example.com/a')]) + assert list(records_by_file(prefix)) == ['out-001.warc.gz'] + + +def test_shared_file_numbers(tmp_path): + numbers = iter([5, 9]) + prefix = str(tmp_path / 'out') + series = WarcFileSeries(prefix, {}, max_file_size=1, next_file_sequence=lambda current: next(numbers)) + write_all(series, [record('https://example.com/a'), record('https://example.com/b')]) + assert list(records_by_file(prefix)) == ['out-005.warc.gz', 'out-009.warc.gz'] + + +def test_failure_aborts_the_current_file_only(tmp_path): + prefix = str(tmp_path / 'out') + series = WarcFileSeries(prefix, {}, max_file_size=1) + original_write = LocalOutputFile.write + + writes = [] + + async def write(self, data): + writes.append(self.file_path) + if writes.count(self.file_path) == 2 and self.file_path.endswith('-002.warc.gz'): # after the warcinfo + raise OSError('disk full') + await original_write(self, data) + + async def main(): + await series.write(record('https://example.com/a')) + try: + await series.write(record('https://example.com/b')) + except OSError: + await series.abort() + raise + + with patch.object(LocalOutputFile, 'write', write), pytest.raises(OSError, match='disk full'): + asyncio.run(main()) + assert list(records_by_file(prefix)) == ['out-001.warc.gz'] diff --git a/tests/repackage/test_sql_index_e2e.py b/tests/repackage/test_sql_index_e2e.py new file mode 100644 index 0000000..384080e --- /dev/null +++ b/tests/repackage/test_sql_index_e2e.py @@ -0,0 +1,135 @@ +"""End-to-end tests of the SQL indexes (DuckDB, Athena): `cdxt index ... | cdxt repackage`. + +By default, DuckDB queries a local columnar index (the local_url_index fixture) and the WARC records are +served by the mocked data.commoncrawl.org (see tests/conftest.py). With CDXT_LIVE_TESTS=s3 or athena, the live +tests query Common Crawl's index on S3 (DuckDB) or with Athena, only a single crawl partition for a single host +(cheap, partition-pruned -- never an all-crawls scan), and fetch the WARC records from s3://commoncrawl. +""" +import json +import os + +import fsspec +import pytest +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main +from tests.conftest import ( + LOCAL_INDEX_CRAWL, LOCAL_INDEX_HOST, TEST_ATHENA_S3_LOCATION, mock_response_from_jsonl, requires_aws_athena, + requires_duckdb, requires_live_s3, +) + + +CRAWL = 'CC-MAIN-2026-17' +HOST = 'commoncrawl.org' +S3_PREFIX = 's3://commoncrawl' +ATHENA = f'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation={TEST_ATHENA_S3_LOCATION}' + + +@pytest.fixture(autouse=True) +def crawl_list_cache(tmp_path, monkeypatch): + """--crawl is resolved with the list of crawls; cache it apart from the list of other tests.""" + monkeypatch.setattr('cdx_toolkit.commoncrawl.CACHE_DIR', str(tmp_path / 'cache') + '/') + + +@pytest.fixture +def mocked(mock_data_commoncrawl): + """The mocked data.commoncrawl.org and the recorded list of crawls (for --crawl).""" + if mock_data_commoncrawl is not None: + mock_response_from_jsonl('test_get_cc_endpoints', 'test_cc') + + +def _index_and_repackage(tmpdir, index_args, host=HOST, warc_download_prefix=None): + captures_path = os.path.join(str(tmpdir), 'captures.jsonl') + main(args=['index'] + index_args + ['-o', captures_path]) + + with open(captures_path) as f: + captures = [json.loads(line) for line in f] + assert len(captures) > 0, 'expected at least one capture for the host/crawl' + assert all(host in c['url'] for c in captures) + + base_prefix = str(tmpdir) + prefix_args = ['--warc-download-prefix', warc_download_prefix] if warc_download_prefix else [] + main(args=['repackage', captures_path, f'--prefix={base_prefix}/TEST_sql'] + prefix_args) + + response_count = 0 + with fsspec.open(os.path.join(base_prefix, 'TEST_sql-001.warc.gz'), 'rb') as stream: + for record in ArchiveIterator(stream): + if record.rec_type == 'response': + response_count += 1 + target = record.rec_headers.get_header('WARC-Target-URI') or '' + assert host in target, f'unexpected target URI: {target}' + + assert response_count == len(captures), 'every capture should yield a response record' + return captures + + +@requires_duckdb +@pytest.mark.usefixtures('mocked') +def test_index_duckdb_local_e2e(tmpdir, local_url_index): + captures = _index_and_repackage( + tmpdir, [str(local_url_index), '--index-backend', 'duckdb', '--crawl', LOCAL_INDEX_CRAWL, + '--hostname', LOCAL_INDEX_HOST], host=LOCAL_INDEX_HOST, + ) + # all captures of the host in the crawl, not those of the other crawl + assert len(captures) == 10 + assert {c['crawl'] for c in captures} == {LOCAL_INDEX_CRAWL} + + +@requires_duckdb +@pytest.mark.usefixtures('mocked') +def test_index_duckdb_local_domain_e2e(tmpdir, local_url_index): + captures = _index_and_repackage( + tmpdir, [str(local_url_index), '--index-backend', 'duckdb', '--crawl', LOCAL_INDEX_CRAWL, + '--domain', 'missiondefrance.fr', '--limit', '5'], host=LOCAL_INDEX_HOST, + ) + assert len(captures) == 5 + + +@requires_duckdb +@pytest.mark.usefixtures('mocked') +def test_index_duckdb_local_sql_e2e(tmpdir, local_url_index): + sql = os.path.join(str(tmpdir), 'query.sql') + with open(sql, 'w') as f: + f.write(f""" + SELECT url, warc_filename, warc_record_offset, warc_record_length, content_languages + FROM read_parquet('{local_url_index}/crawl={LOCAL_INDEX_CRAWL}/subset=warc/*.parquet') + WHERE url_host_name = '{LOCAL_INDEX_HOST}' + LIMIT 3 + """) + captures = _index_and_repackage(tmpdir, [str(local_url_index), '--index-backend', 'duckdb', '--sql', sql, + '--yes'], host=LOCAL_INDEX_HOST) + assert len(captures) == 3 + assert {c['languages'] for c in captures} == {'fra'} + + +@requires_aws_athena +def test_index_athena_e2e(tmpdir): + _index_and_repackage(tmpdir, [ATHENA, '--crawl', CRAWL, '--hostname', HOST], warc_download_prefix=S3_PREFIX) + + +@requires_live_s3 +@requires_duckdb +def test_index_duckdb_e2e(tmpdir): + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--hostname', HOST], + warc_download_prefix=S3_PREFIX) + + +@requires_live_s3 +@requires_duckdb +def test_index_duckdb_domain_e2e(tmpdir): + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--crawl', CRAWL, '--domain', HOST, '--limit', '20'], + warc_download_prefix=S3_PREFIX) + + +@requires_live_s3 +@requires_duckdb +def test_index_duckdb_sql_e2e(tmpdir): + sql = os.path.join(str(tmpdir), 'query.sql') + with open(sql, 'w') as f: + f.write(f""" + SELECT url, warc_filename, warc_record_offset, warc_record_length, content_languages + FROM read_parquet('s3://commoncrawl/cc-index/table/cc-main/warc/crawl={CRAWL}/subset=warc/*.parquet') + WHERE url_host_name = '{HOST}' + LIMIT 10 + """) + _index_and_repackage(tmpdir, ['cc-url-index-s3', '--sql', sql, '--yes'], warc_download_prefix=S3_PREFIX) diff --git a/tests/repackage/test_warc_from_fs.py b/tests/repackage/test_warc_from_fs.py new file mode 100644 index 0000000..eb9da18 --- /dev/null +++ b/tests/repackage/test_warc_from_fs.py @@ -0,0 +1,56 @@ +import pytest + +from tests.conftest import requires_aws_s3 +from cdx_toolkit.warc import fetch_warc_record + + +@pytest.mark.usefixtures('mock_data_commoncrawl') +def test_fetch_warc_record_from_http(): + encoding = 'utf-8' + capture = { + 'url': 'https://bibliotheque.missiondefrance.fr/index.php?lvl=bulletin_display&id=319', + 'mime': 'text/html', + 'mime-detected': 'application/xhtml+xml', + 'status': '200', + 'digest': 'D5K3FUWDRAOMMTJC2CTWV7L2ABFIJ5BP', + 'length': '9754', + 'offset': '111440525', + 'filename': 'crawl-data/CC-MAIN-2024-30/segments/1720763514759.37/warc/CC-MAIN-20240716142214-20240716172214-00337.warc.gz', # noqa: E501 + 'charset': 'UTF-8', + 'languages': 'fra', + 'timestamp': '20240716153155', + } + warc_download_prefix = 'https://data.commoncrawl.org' + + record = fetch_warc_record(capture, warc_download_prefix) + record_content = record.content_stream().read().decode(encoding, errors='ignore') + + assert record.rec_type == 'response' + assert record.length == 75825 + assert 'Catalogue en ligne Mission de France' in record_content + + +@requires_aws_s3 +def test_fetch_warc_record_from_s3(s3_env): + encoding = 'utf-8' + capture = { + 'url': 'https://bibliotheque.missiondefrance.fr/index.php?lvl=bulletin_display&id=319', + 'mime': 'text/html', + 'mime-detected': 'application/xhtml+xml', + 'status': '200', + 'digest': 'D5K3FUWDRAOMMTJC2CTWV7L2ABFIJ5BP', + 'length': '9754', + 'offset': '111440525', + 'filename': 'crawl-data/CC-MAIN-2024-30/segments/1720763514759.37/warc/CC-MAIN-20240716142214-20240716172214-00337.warc.gz', # noqa: E501 + 'charset': 'UTF-8', + 'languages': 'fra', + 'timestamp': '20240716153155', + } + warc_download_prefix = 's3://commoncrawl' + + record = fetch_warc_record(s3_env.capture(capture), warc_download_prefix) + record_content = record.content_stream().read().decode(encoding, errors='ignore') + + assert record.rec_type == 'response' + assert record.length == 75825 + assert 'Catalogue en ligne Mission de France' in record_content diff --git a/tests/repackage/test_warc_records.py b/tests/repackage/test_warc_records.py new file mode 100644 index 0000000..f04fa23 --- /dev/null +++ b/tests/repackage/test_warc_records.py @@ -0,0 +1,39 @@ +import fsspec +import pytest + +from cdx_toolkit.repackage.warc_records import get_metadata_record_from_path +from cdx_toolkit.repackage.writers.series import existing_warc_files +from tests.conftest import TEST_DATA_PATH + + +def test_get_metadata_record_from_path(): + file_path = TEST_DATA_PATH / 'repackage/url_list.txt' + record = get_metadata_record_from_path(file_path, warcinfo_id="abc123") + + assert record.content_type == 'text/plain' + + record_headers = dict(record.rec_headers.headers) + + assert record_headers["WARC-Warcinfo-ID"] == "abc123", "Invalid Warcinfo-ID" + assert record_headers["WARC-Payload-Digest"] == "sha1:VXA2A5YUS3TAY36AUO6MACRMNOH5RXG2", "Invalid payload digest" + + +@pytest.mark.parametrize('subprefix, expected', [ + (None, ['out-000000.extracted.warc.gz', 'out-001.warc.gz', 'out-1000.warc.gz']), + ('SUB', ['out-SUB-002.warc.gz']), + ('OTHER', []), +]) +def test_existing_warc_files(tmp_path, subprefix, expected): + for name in ['out-001.warc.gz', 'out-1000.warc.gz', 'out-000000.extracted.warc.gz', 'out-x.extracted.warc.gz', + 'out-SUB-002.warc.gz', 'out-x.warc.gz', 'out-001.warc', 'outer-001.warc.gz', 'other-001.warc.gz']: + (tmp_path / name).write_bytes(b'') + found = existing_warc_files(str(tmp_path / 'out'), subprefix) + assert [f.rsplit('/', 1)[-1] for f in found] == expected + + +def test_existing_warc_files_remote_and_missing_dir(tmp_path): + with fsspec.open('memory://existing/b[1]/out-003.warc.gz', 'wb') as f: + f.write(b'') + found, = existing_warc_files('memory://existing/b[1]/out') + assert found.endswith('existing/b[1]/out-003.warc.gz') + assert existing_warc_files(str(tmp_path / 'missing' / 'out')) == [] diff --git a/tests/repackage/test_warc_writer.py b/tests/repackage/test_warc_writer.py new file mode 100644 index 0000000..cecea40 --- /dev/null +++ b/tests/repackage/test_warc_writer.py @@ -0,0 +1,144 @@ +from io import BytesIO +import fsspec +import pytest +import cdx_toolkit + +from tests.conftest import TEST_DATA_PATH, requires_aws_s3 + +from warcio import WARCWriter +from warcio.archiveiterator import ArchiveIterator + + +fixture_path = TEST_DATA_PATH / 'repackage' + + +@pytest.mark.parametrize( + 'prefix,gzip', + [ + pytest.param('test-prefix', False, id='File name prefix on local'), + pytest.param('test-prefix', True, id='File name prefix on local with gzip'), + # raised FileNotFound error (parent dir does not exist) + # pytest.param("test-prefix-folder/file-prefix", None, id="Folder as prefix"), + ], +) +def test_write_to_local(prefix, gzip, tmpdir): + info = { + 'software': 'pypi_cdx_toolkit/test', + 'description': 'test', + 'format': 'WARC file version 1.0', + } + encoding = 'utf-8' + full_prefix = str(tmpdir) + '/' + prefix + fs, fs_prefix_path = fsspec.url_to_fs(full_prefix) + + with pytest.warns(FutureWarning, match='deprecated'): + writer = cdx_toolkit.warc.get_writer(full_prefix, None, info, gzip=gzip) + + # single record + input_resource_record_text = 'foo bar text' + writer.write_record( + WARCWriter(None).create_warc_record( + uri='foo/bar', + record_type='resource', + payload=BytesIO(input_resource_record_text.encode(encoding)), + warc_content_type='text/plain', + ) + ) + writer.close() + + # Check that WARC file was created + warc_path = fs_prefix_path + '-000000.extracted.warc' + if gzip: + warc_path += '.gz' + + assert fs.exists(warc_path) + + # Validate that creator/operator are not in warcinfo record + info_record = None + resource_record = None + with open(warc_path, 'rb') as stream: + for record in ArchiveIterator(stream): + if record.rec_type == 'warcinfo': + info_record = record.content_stream().read().decode(encoding) + + if record.rec_type == 'resource': + resource_record = record.content_stream().read().decode(encoding) + break + + assert resource_record is not None + assert info_record is not None + + assert 'description: test' in info_record + assert resource_record == input_resource_record_text + + +@requires_aws_s3 +def test_write_to_s3(s3_tmpdir): + info = { + 'software': 'pypi_cdx_toolkit/test', + 'description': 'test', + 'format': 'WARC file version 1.0', + } + encoding = 'utf-8' + + fs, fs_prefix_path = fsspec.url_to_fs(s3_tmpdir) + + with pytest.warns(FutureWarning, match='deprecated'): + writer = cdx_toolkit.warc.get_writer(s3_tmpdir, None, info) + + # single record + input_resource_record_text = 'foo bar text' + writer.write_record( + WARCWriter(None).create_warc_record( + uri='foo/bar', + record_type='resource', + payload=BytesIO(input_resource_record_text.encode(encoding)), + warc_content_type='text/plain', + ) + ) + writer.close() + + # Check that WARC file was created + warc_path = fs_prefix_path + '-000000.extracted.warc.gz' + assert fs.exists(warc_path) + + # Validate that creator/operator are not in warcinfo record + info_record = None + resource_record = None + with fs.open(warc_path, 'rb') as stream: + for record in ArchiveIterator(stream): + if record.rec_type == 'warcinfo': + info_record = record.content_stream().read().decode(encoding) + + if record.rec_type == 'resource': + resource_record = record.content_stream().read().decode(encoding) + break + + assert resource_record is not None + assert info_record is not None + + assert 'description: test' in info_record + assert resource_record == input_resource_record_text + + +def test_warc_info(): + warc_version = '1.0' + gzip = False + file_handler = BytesIO() + filename = 'foo.warc' + + info = { + 'software': 'pypi_cdx_toolkit/123', + 'isPartOf': 'bar', + 'description': 'warc extraction based on CDX generated with: xx', + 'format': 'WARC file version 1.0', + } + + writer = WARCWriter(file_handler, gzip=gzip, warc_version=warc_version) + warcinfo = writer.create_warcinfo_record(filename, info) + + writer.write_record(warcinfo) + + file_value = file_handler.getvalue().decode('utf-8') + + assert 'pypi_cdx_toolkit/123' in file_value diff --git a/tests/test_cli.py b/tests/test_cli.py index 5d385f8..a0addbb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,8 +3,11 @@ import os import platform import logging +import re import pytest +import requests +import responses from cdx_toolkit.cli import main @@ -19,8 +22,8 @@ def slow_ci(): For Github Actions, the windows and macos runners are very slow. Detect those runners, so that we can cut testing short. """ - if os.environ.get('FAKE_GITHUB_ACTION'): - LOGGER.error('limiting pytest because FAKE_GITHUB_ACTION') + if os.environ.get('CDXT_FAKE_GITHUB_ACTION'): + LOGGER.error('limiting pytest because CDXT_FAKE_GITHUB_ACTION') return True if os.environ.get('GITHUB_ACTION'): if platform.system() in {'Darwin', 'Windows'}: @@ -35,7 +38,7 @@ def slow_ci(): @conditional_mock_responses def test_basics(capsys): - args = '--cc --limit 10 iter commoncrawl.org/*'.split() + args = 'index cc --limit 10 --url commoncrawl.org/*'.split() main(args=args) out, err = capsys.readouterr() @@ -48,7 +51,7 @@ def test_basics(capsys): @conditional_mock_responses def test_basics_2(capsys): - args = '--crawl 2 --limit 10 iter commoncrawl.org/*'.split() + args = 'index cc --crawl 2 --limit 10 --url commoncrawl.org/*'.split() main(args=args) out, err = capsys.readouterr() @@ -60,9 +63,7 @@ def test_basics_2(capsys): def multi_helper(t, capsys, caplog): - inputs = t[0] - outputs = t[1] - cmdline = '{} {} {} {}'.format(inputs['service'], inputs['mods'], inputs['cmd'], inputs['rest']) + cmdline, outputs = t args = cmdline.split() if 'exception' in outputs: @@ -97,442 +98,300 @@ def multi_helper(t, capsys, caplog): @conditional_mock_responses def test_multi_cc1(capsys, caplog): # this is the test case before slow_ci -> break - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrep': 'commoncrawl.org'}, - ] + testdata = ('index cc --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': 'commoncrawl.org'}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_1(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 11', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 11, 'linefgrep': 'commoncrawl.org'}, - ] + testdata = ('index cc --limit 11 --url commoncrawl.org/*', {'count': 11, 'linefgrep': 'commoncrawl.org'}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_2(capsys, caplog): - testdata = [ - { - 'service': '--crawl 1', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/thisurlneverdidexist', - }, - {'count': 0}, - ] # runs slowly if we don't limit crawl to 1 + # runs slowly if we don't limit crawl to 1 + testdata = ('index cc --crawl 1 --limit 10 --url commoncrawl.org/thisurlneverdidexist', {'count': 0}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_3(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--cc-mirror https://index.commoncrawl.org/ --limit 11', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, + testdata = ( + 'index cc --cc-mirror https://index.commoncrawl.org/ --limit 11 --url commoncrawl.org/*', {'count': 11, 'linefgrep': 'commoncrawl.org'}, - ] + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_4(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/* --all-fields', - }, - {'count': 10, 'linefgrep': 'digest '}, - ] + # all fields are output by default + testdata = ('index cc --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': '"digest": '}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_5(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/* --fields=digest,length,offset --csv', - }, + testdata = ( + 'index cc --limit 10 --url commoncrawl.org/* --fields digest,length,offset --output-format csv', {'count': 11, 'csv': True}, - ] + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_6(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10 --filter=status:200', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrep': 'status 200'}, - ] + testdata = ( + 'index cc --limit 10 --filter=status:200 --url commoncrawl.org/*', + {'count': 10, 'linefgrep': '"status": "200"'}, + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_7(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10 --filter=!status:200 --filter=!status:404', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrepv': 'status 200'}, - ] + testdata = ( + 'index cc --limit 10 --filter=!status:200 --filter=!status:404 --url commoncrawl.org/*', + {'count': 10, 'linefgrepv': '"status": "200"'}, + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_8(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10 --to=2017', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrep': 'timestamp 2017'}, - ] + testdata = ( + 'index cc --limit 10 --to=2017 --url commoncrawl.org/*', + {'count': 10, 'linefgrep': '"timestamp": "2017'}, + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc1_slow_9(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10 --from=2017 --to=2017', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'linefgrep': 'timestamp 2017'}, - ] + testdata = ( + 'index cc --limit 10 --from=2017 --to=2017 --url commoncrawl.org/*', + {'count': 10, 'linefgrep': '"timestamp": "2017'}, + ) multi_helper(testdata, capsys, caplog) @conditional_mock_responses def test_multi_cc2(capsys, caplog): # this is the test case before slow_ci -> break - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 3 --get --closest=20170615', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 3, 'linefgrep': 'timestamp 20170'}, - ] # data-dependent, and kinda broken + testdata = ( + 'index cc --limit 3 --closest=20170615 --url commoncrawl.org/*', + {'count': 3, 'linefgrep': '"timestamp": "20170'}, + ) # data-dependent, and kinda broken multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_1(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 3 --get --filter status:200 --closest=20170615', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 3, 'linefgrep': 'timestamp 20170'}, - ] # data-dependent, and kinda broken + testdata = ( + 'index cc --limit 3 --filter status:200 --closest=20170615 --url commoncrawl.org/*', + {'count': 3, 'linefgrep': '"timestamp": "20170'}, + ) # data-dependent, and kinda broken multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_2(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--get --closest=20170615', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/never-existed', - }, - {'count': 0}, - ] + testdata = ('index cc --closest=20170615 --url commoncrawl.org/never-existed', {'count': 0}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_3(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/* --csv', - }, - {'count': 11, 'csv': True}, - ] + testdata = ('index cc --limit 10 --url commoncrawl.org/* --output-format csv', {'count': 11, 'csv': True}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_4(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/* --jsonl', - }, - {'count': 10, 'jsonl': True}, - ] + testdata = ('index cc --limit 10 --url commoncrawl.org/* --output-format jsonl', {'count': 10, 'jsonl': True}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_cc2_slow_5(capsys, caplog): - testdata = [ - { - 'service': '--cc', - 'mods': '-v -v --limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, - {'count': 10, 'debug': 5}, - ] + testdata = ('index cc -v -v --limit 10 --url commoncrawl.org/*', {'count': 10, 'debug': 5}) multi_helper(testdata, capsys, caplog) @conditional_mock_responses def test_multi_ia(capsys, caplog): - testdata = [ - {'service': '--ia', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - {'count': 10, 'linefgrep': 'commoncrawl.org'}, - ] + testdata = ('index ia --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': 'commoncrawl.org'}) # Disabled: minimize IA for ratelimit purposes - # [{'service': '--ia', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/thisurlneverdidexist'}, - # {'count': 0}], - # [{'service': '--ia', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/* --all-fields'}, - # {'count': 10, 'linefgrep': 'mime ', 'linefgrepv': 'original '}], # both of these are renamed fields - # [{'service': '--ia', 'mods': '--get --limit 4 --closest=20170615', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - # {'count': 4, 'linefgrep': 'timestamp '}], # returns 2008 ?! bug probably on my end - # [{'service': '--ia', 'mods': '-v -v --limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - # {'count': 10, 'debug': 5}] + # ('index ia --limit 10 --url commoncrawl.org/thisurlneverdidexist', {'count': 0}), + # ('index ia --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': '"mime": '}), # renamed field + # ('index ia --limit 4 --closest=20170615 --url commoncrawl.org/*', + # {'count': 4, 'linefgrep': '"timestamp": '}), # returns 2008 ?! bug probably on my end + # ('index ia -v -v --limit 10 --url commoncrawl.org/*', {'count': 10, 'debug': 5}), multi_helper(testdata, capsys, caplog) +@responses.activate def test_multi_misc_not_ia(capsys, caplog): - # this is the test case before slow_ci -> break - testdata = [ - { - 'service': '-v -v --source https://web.arc4567hive.org/cdx/search/cdx', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, + # this is the test case before slow_ci -> break; the hostname does not exist (no DNS request: mocked) + dns_error = requests.exceptions.ConnectionError( + "HTTPSConnectionPool(host='web.arc4567hive.org', port=443): Failed to resolve 'web.arc4567hive.org' " + '([Errno 8] nodename nor servname provided, or not known)' + ) + responses.add(responses.GET, re.compile(r'https://web\.arc4567hive\.org/.*'), body=dns_error) + testdata = ( + '-v -v index https://web.arc4567hive.org/cdx/search/cdx --limit 10 --url commoncrawl.org/*', {'exception': ValueError}, - ] + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_misc_not_ia_slow_1(capsys, caplog): - testdata = [ - { - 'service': '-v -v --source https://example.com/404', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, + testdata = ( + '-v -v index https://example.com/404 --limit 10 --url commoncrawl.org/*', {'exception': ValueError}, - ] + ) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_misc_not_ia_slow_2(capsys, caplog): - testdata = [ - {'service': '--crawl 1,1', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - {'exception': ValueError}, - ] + testdata = ('index cc --crawl 1,1 --limit 10 --url commoncrawl.org/*', {'exception': ValueError}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_misc_not_ia_slow_3(capsys, caplog): - testdata = [ - {'service': '--crawl 1,CC-MAIN-2024', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - {'exception': ValueError}, - ] + testdata = ('index cc --crawl 1,CC-MAIN-2024 --limit 10 --url commoncrawl.org/*', {'exception': ValueError}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_multi_misc_not_ia_slow_4(capsys, caplog): - testdata = [ - {'service': '--cc', 'mods': '--limit 10', 'cmd': 'size', 'rest': 'commoncrawl.org/*'}, - {'count': 1, 'is_int': True}, - ] - multi_helper(testdata, capsys, caplog) - - -@pytest.mark.skipif(slow_ci(), reason='Slow CI') -@conditional_mock_responses -def test_multi_misc_not_ia_slow_5(capsys, caplog): - testdata = [ - {'service': '--cc', 'mods': '--limit 10', 'cmd': 'size', 'rest': '--details commoncrawl.org/*'}, - {'count': 2}, - ] + testdata = ('index cc --limit 10 --url commoncrawl.org/* --count', {'count': 1, 'is_int': True}) multi_helper(testdata, capsys, caplog) @pytest.mark.skipif(slow_ci(), reason='Slow CI') -@conditional_mock_responses def test_multi_misc_not_ia_slow_6(capsys, caplog): - testdata = [ - {'service': '', 'mods': '--limit 10', 'cmd': 'iter', 'rest': 'commoncrawl.org/*'}, - {'exception': ValueError}, - ] - multi_helper(testdata, capsys, caplog) + # INDEX is required + with pytest.raises(SystemExit): + main(args='index --limit 10 --url commoncrawl.org/*'.split()) @conditional_mock_responses def test_multi_misc_ia(capsys, caplog): - testdata = [ - { - 'service': '--source https://web.archive.org/cdx/search/cdx', - 'mods': '--limit 10', - 'cmd': 'iter', - 'rest': 'commoncrawl.org/*', - }, + testdata = ( + 'index https://web.archive.org/cdx/search/cdx --limit 10 --url commoncrawl.org/*', {'count': 10, 'linefgrep': 'commoncrawl.org'}, - ] + ) # Disabled: minimize IA for ratelimit reasons - # [{'service': '--ia', 'mods': '--limit 10', 'cmd': 'size', 'rest': 'commoncrawl.org/*'}, - # {'count': 1, 'is_int': True}], - # [{'service': '--ia', 'mods': '--limit 10', 'cmd': 'size', 'rest': '--details commoncrawl.org/*'}, - # {'count': 2}], - # [{'service': '--ia', 'mods': '--from 20180101 --to 20180110 --limit 10', 'cmd': 'size', - # 'rest': '--details commoncrawl.org'}, - # {'count': 2}], + # ('index ia --limit 10 --url commoncrawl.org/* --count', {'count': 1, 'is_int': True}), + # ('index ia --from 20180101 --to 20180110 --limit 10 --url commoncrawl.org --count', {'count': 1}), multi_helper(testdata, capsys, caplog) -def warc_prefix_test_helper(tmpdir, prefix: str): - # crash testing only, so far - base = ' --limit 1 warc commoncrawl.org/*' - +def warc_test_helper(tmpdir, index_args: str, repackage_args: str = ''): with tmpdir.as_cwd(): - cmdline = prefix + base - if 'cc' in cmdline: - cmdline = cmdline.replace(' 1', ' 2') - print(cmdline, file=sys.stderr) - args = cmdline.split() - main(args=args) + for cmdline in ( + f'{index_args} -o captures.jsonl', + f'repackage captures.jsonl {repackage_args}', + ): + print(cmdline, file=sys.stderr) + main(args=cmdline.split()) @conditional_mock_responses def test_warc_prefix_1(tmpdir): - warc_prefix_test_helper(tmpdir, '-v -v --cc') + warc_test_helper(tmpdir, '-v -v index cc --limit 2 --url commoncrawl.org/*') + assert tmpdir.listdir(fil='REPACKAGE-001.warc.gz') @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_prefix_2(tmpdir): - warc_prefix_test_helper(tmpdir, '--ia') + warc_test_helper(tmpdir, 'index ia --limit 1 --url commoncrawl.org/*', '--wb ia') + assert tmpdir.listdir(fil='REPACKAGE-*.warc.gz') @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_prefix_3(tmpdir): - warc_prefix_test_helper(tmpdir, '--cc --cc-mirror https://index.commoncrawl.org/') + warc_test_helper(tmpdir, 'index cc --cc-mirror https://index.commoncrawl.org/ --limit 2 --url commoncrawl.org/*') + assert tmpdir.listdir(fil='REPACKAGE-001.warc.gz') @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_prefix_4(tmpdir): - warc_prefix_test_helper(tmpdir, '--source https://web.archive.org/cdx/search/cdx --wb https://web.archive.org/web') - - -def warc_suffix_test_helper(tmpdir, suffix: str): - # crash testing only, so far - base = ' --limit 1 warc commoncrawl.org/*' - prefix = '-v -v --cc' - - with tmpdir.as_cwd(): - cmdline = prefix + base + ' ' + suffix - print(cmdline, file=sys.stderr) - args = cmdline.split() - main(args=args) + warc_test_helper( + tmpdir, + 'index https://web.archive.org/cdx/search/cdx --limit 1 --url commoncrawl.org/*', + '--wb https://web.archive.org/web', + ) + assert tmpdir.listdir(fil='REPACKAGE-*.warc.gz') @conditional_mock_responses def test_warc_suffix_1(tmpdir): - warc_suffix_test_helper( + warc_test_helper( tmpdir, - '--prefix FOO --subprefix BAR --size 1 --creator creator --operator bob --url-fgrep common --url-fgrepv bar', + '-v -v index cc --limit 1 --url-fgrep common --url-fgrepv bar --url commoncrawl.org/*', + '--prefix FOO --subprefix BAR --size 1 --creator creator --operator bob', ) + assert tmpdir.listdir(fil='FOO-BAR-001.warc.gz') @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_suffix_2(tmpdir): - warc_suffix_test_helper(tmpdir, '--prefix EMPTY --size 1 --url-fgrep bar') + warc_test_helper( + tmpdir, '-v -v index cc --limit 1 --url-fgrep bar --url commoncrawl.org/*', '--prefix EMPTY --size 1' + ) + assert not tmpdir.listdir(fil='EMPTY-*') # no WARC file without records @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_suffix_3(tmpdir): - warc_suffix_test_helper(tmpdir, '--prefix EMPTY --size 1 --url-fgrepv common') + warc_test_helper( + tmpdir, '-v -v index cc --limit 1 --url-fgrepv common --url commoncrawl.org/*', '--prefix EMPTY --size 1' + ) + assert not tmpdir.listdir(fil='EMPTY-*') # no WARC file without records @pytest.mark.skipif(slow_ci(), reason='Slow CI') @conditional_mock_responses def test_warc_suffix_4(tmpdir): - warc_suffix_test_helper( + warc_test_helper( tmpdir, - '--prefix FOO --subprefix BAR --size 1 --creator creator --operator bob --url-fgrep common --url-fgrepv bar', + '-v -v index cc --limit 1 --url-fgrep common --url-fgrepv bar --url commoncrawl.org/*', + '--prefix FOO --subprefix BAR --size 1 --creator creator --operator bob', ) + assert tmpdir.listdir(fil='FOO-BAR-001.warc.gz') -def one_ia_corner(tmpdir, cmdline): - with tmpdir.as_cwd(): - main(args=cmdline.split()) +def one_ia_corner(tmpdir, index_args): + warc_test_helper(tmpdir, 'index ia ' + index_args, '--wb ia') @pytest.mark.skip(reason='needs some ratelimit love') @@ -543,17 +402,15 @@ def test_warc_ia_corners(tmpdir, caplog): """ # revisit vivification - cmdline = '--ia --from 2017010118350 --to 2017010118350 warc pbm.com/robots.txt' - one_ia_corner(tmpdir, cmdline) + one_ia_corner(tmpdir, '--from 2017010118350 --to 2017010118350 --url pbm.com/robots.txt') # same-surt same-timestamp redir+200 - cmdline = '--ia --from 20090220001146 --to 20090220001146 warc pbm.com' - one_ia_corner(tmpdir, cmdline) + one_ia_corner(tmpdir, '--from 20090220001146 --to 20090220001146 --url pbm.com') # any redir -> 302, this is a 301 - cmdline = '--ia --from 2011020713024 --to 2011020713024 warc pbm.com' - one_ia_corner(tmpdir, cmdline) + one_ia_corner(tmpdir, '--from 2011020713024 --to 2011020713024 --url pbm.com') # warcing a 404 is a corner case in myrequests - cmdline = '--ia --from 20080512074145 --to 20080512074145 warc http://www.pbm.com/oly/archive/design94/0074.html' - one_ia_corner(tmpdir, cmdline) + one_ia_corner( + tmpdir, '--from 20080512074145 --to 20080512074145 --url http://www.pbm.com/oly/archive/design94/0074.html' + ) diff --git a/tests/test_cli_legacy.py b/tests/test_cli_legacy.py new file mode 100644 index 0000000..8a929d2 --- /dev/null +++ b/tests/test_cli_legacy.py @@ -0,0 +1,192 @@ +'''The command lines of cdx_toolkit 0.9.39 (deprecated), with the responses recorded for them: the mock data of +tests/test_cli.py was recorded with these command lines (`--cc --limit 10 iter URL` makes the same requests as +`index cc --limit 10 --url URL`).''' +import functools +import json +import re + +import pytest +import responses +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main + +from tests.conftest import LIVE_TESTS, conditional_mock_responses, mock_response_from_jsonl +from tests.test_cli import slow_ci + + +DEPRECATED = 'this command line uses the old syntax, which is deprecated' +TEXT_LINE = re.compile(r'status \d+, timestamp \d{14}, url \S+') + + +def recorded(name): + """The recorded responses of tests/test_cli.py:: (unless CDXT_LIVE_TESTS has http).""" + def decorator(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + if 'http' not in LIVE_TESTS: + mock_response_from_jsonl(name, 'test_cli') + return f(*args, **kwargs) + + return conditional_mock_responses(auto_mock_data=False)(wrapper) + + return decorator + + +def run(capsys, cmdline): + main(args=cmdline.split()) + out, err = capsys.readouterr() + assert DEPRECATED in err and 'suggested replacement: cdxt ' in err + return out, err + + +@recorded('test_basics') +def test_basics(capsys): + out, err = run(capsys, '--cc --limit 10 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 10 + for line in lines: + assert TEXT_LINE.fullmatch(line) and 'commoncrawl.org' in line + assert ("cdxt index cc --url 'commoncrawl.org/*' --limit 10 --output-format csv --fields status,timestamp,url" + in err) + + +@recorded('test_basics_2') +def test_basics_2(capsys): + out, _ = run(capsys, '--crawl 2 --limit 10 iter commoncrawl.org/*') + assert len(out.splitlines()) == 10 + + +@recorded('test_multi_cc1_slow_4') +def test_all_fields(capsys): + out, _ = run(capsys, '--cc --limit 10 iter commoncrawl.org/* --all-fields') + lines = out.splitlines() + assert len(lines) == 10 and all('digest ' in line and 'filename crawl-data/' in line for line in lines) + + +@recorded('test_multi_cc1_slow_5') +def test_csv(capsys): + out, _ = run(capsys, '--cc --limit 10 iter commoncrawl.org/* --fields=digest,length,offset,digest --csv') + lines = out.split('\r\n') + assert lines[0] == 'digest,length,offset' + assert len(lines) == 12 and lines[-1] == '' # header, 10 captures, CRLF line endings + assert all(line.count(',') == 2 for line in lines[:-1]) + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_cc2_slow_4') +def test_jsonl(capsys): + out, _ = run(capsys, '--cc --limit 10 iter commoncrawl.org/* --jsonl') + lines = out.splitlines() + assert len(lines) == 10 + assert all(list(json.loads(line)) == ['status', 'timestamp', 'url'] for line in lines) + + +@recorded('test_multi_cc1_slow_7') +def test_filters(capsys): + out, _ = run(capsys, '--cc --limit 10 --filter=!status:200 --filter=!status:404 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 10 and not any(line.startswith(('status 200,', 'status 404,')) for line in lines) + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_cc1_slow_9') +def test_from_to(capsys): + out, _ = run(capsys, '--cc --limit 10 --from=2017 --to=2017 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 10 and all(', timestamp 2017' in line for line in lines) + + +@recorded('test_multi_cc2') +def test_get_closest(capsys): + out, _ = run(capsys, '--cc --limit 3 --get --closest=20170615 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 3 and all(', timestamp 20170' in line for line in lines) + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_cc2_slow_2') +def test_get_without_limit(capsys): + # the recorded requests have limit=1000, the default of a single get + out, err = run(capsys, '--cc --get --closest=20170615 iter commoncrawl.org/never-existed') + assert out == '' + assert '--limit 1000 --closest 20170615' in err + + +@recorded('test_multi_ia') +def test_ia(capsys): + out, _ = run(capsys, '--ia --limit 10 iter commoncrawl.org/*') + assert len(out.splitlines()) == 10 + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_misc_ia') +def test_source(capsys): + out, _ = run(capsys, '--source https://web.archive.org/cdx/search/cdx --limit 10 iter commoncrawl.org/*') + assert len(out.splitlines()) == 10 + + +@recorded('test_multi_misc_not_ia_slow_4') +def test_size(capsys): + out, err = run(capsys, '--cc --limit 10 size commoncrawl.org/*') + assert out.strip().isdigit() + assert "cdxt index cc --url 'commoncrawl.org/*' --limit 10 --count" in err + + +@responses.activate +def test_size_details(capsys): + # one CDX API server: the estimate of the index, then the total (not capped by --limit) + responses.add(responses.GET, 'https://example.com/cdx', json={'blocks': 3}) + out, _ = run(capsys, '--source https://example.com/cdx --limit 10 size --details commoncrawl.org/*') + assert out == 'https://example.com/cdx 6000\n6000\n' + assert responses.calls[0].request.params == {'url': 'commoncrawl.org/*', 'showNumPages': 'true', 'limit': '10'} + + +def warc_records(path): + """(type, content) of the records of a WARC file.""" + with open(path, 'rb') as f: + return [(record.rec_type, record.content_stream().read().decode(errors='replace')) + for record in ArchiveIterator(f)] + + +@recorded('test_warc_prefix_1') +def test_warc_cc(tmpdir, capsys): + with tmpdir.as_cwd(): + _, err = run(capsys, '-v -v --cc --limit 2 warc commoncrawl.org/*') + assert [p.basename for p in tmpdir.listdir()] == ['TEST-001.warc.gz'] + records = warc_records(tmpdir / 'TEST-001.warc.gz') + assert [rec_type for rec_type, _ in records] == ['warcinfo', 'response', 'response'] + warcinfo = records[0][1] + assert 'isPartOf: TEST' in warcinfo + assert 'description: warc extraction generated with: cdxt -v -v --cc --limit 2 warc commoncrawl.org/*' in warcinfo + assert "cdxt index cc --url 'commoncrawl.org/*' --limit 2 | cdxt repackage --prefix TEST --no-sort" in err + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_warc_prefix_2') +def test_warc_ia(tmpdir, capsys): + # replayed from the Internet Archive's wayback machine, without --wb + with tmpdir.as_cwd(): + _, err = run(capsys, '--ia --limit 1 warc commoncrawl.org/*') + assert [p.basename for p in tmpdir.listdir()] == ['TEST-001.warc.gz'] + assert [rec_type for rec_type, _ in warc_records(tmpdir / 'TEST-001.warc.gz')] == ['warcinfo', 'response'] + assert '| cdxt repackage --prefix TEST --wb ia --no-sort' in err + + +@recorded('test_warc_suffix_1') +def test_warc_options(tmpdir, capsys): + with tmpdir.as_cwd(): + run(capsys, '-v -v --cc --limit 1 warc --prefix FOO --subprefix BAR --size 1 --creator creator ' + '--operator bob --url-fgrep common --url-fgrepv bar commoncrawl.org/*') + assert [p.basename for p in tmpdir.listdir()] == ['FOO-BAR-001.warc.gz'] + records = warc_records(tmpdir / 'FOO-BAR-001.warc.gz') + warcinfo = records[0][1] + assert 'isPartOf: FOO-BAR' in warcinfo and 'creator: creator' in warcinfo and 'operator: bob' in warcinfo + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_warc_suffix_2') +def test_warc_empty_selection(tmpdir, capsys): + with tmpdir.as_cwd(): + run(capsys, '-v -v --cc --limit 1 warc --prefix EMPTY --size 1 --url-fgrep bar commoncrawl.org/*') + assert tmpdir.listdir() == [] # no WARC file without captures diff --git a/tests/unit/test_athena.py b/tests/unit/test_athena.py new file mode 100644 index 0000000..a4da261 --- /dev/null +++ b/tests/unit/test_athena.py @@ -0,0 +1,179 @@ +"""Running Athena queries and the Athena index backend with a fake boto3 Athena client (no AWS).""" +import json +import logging +from unittest.mock import patch + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.indexes import IndexQuery +from cdx_toolkit.indexes.backends.athena import AthenaBackend, AthenaClient, make_client + +ATHENA = 'athena://athena.eu-west-1.amazonaws.com/mydb?S3OutputLocation=s3://bucket/athena/' +COLUMNS = ['url', 'fetch_time', 'fetch_status', 'warc_filename', 'warc_record_offset', 'warc_record_length'] +ROWS = [ + ['https://example.com/', '2024-07-13 10:22:33.000', '200', 'crawl-data/a.warc.gz', '10', '100'], + ['https://example.com/x', '2024-07-14 08:00:00.000', '404', 'crawl-data/b.warc.gz', '20', '200'], +] + + +class FakeAthenaClient: + """start_query_execution / get_query_execution (states over time) / paginated results.""" + + def __init__(self, states=('RUNNING', 'SUCCEEDED'), rows=ROWS, scanned=2 * 10 ** 9, reason=None): + self.states = list(states) + self.rows = rows + self.scanned = scanned + self.reason = reason + self.started = [] + self.stopped = [] + + def start_query_execution(self, **kwargs): + self.started.append(kwargs) + return {'QueryExecutionId': 'qid'} + + def get_query_execution(self, QueryExecutionId): + state = self.states.pop(0) if len(self.states) > 1 else self.states[0] + execution = {'Status': {'State': state}} + if self.reason: + execution['Status']['StateChangeReason'] = self.reason + if self.scanned is not None: + execution['Statistics'] = {'DataScannedInBytes': self.scanned} + return {'QueryExecution': execution} + + def stop_query_execution(self, QueryExecutionId): + self.stopped.append(QueryExecutionId) + + def get_paginator(self, name): + assert name == 'get_query_results' + header = {'Data': [{'VarCharValue': c} for c in COLUMNS]} + data = [{'Data': [{'VarCharValue': v} for v in row]} for row in self.rows] + # two pages: the header is only on the first one + pages = [{'ResultSet': {'Rows': [header] + data[:1]}}, {'ResultSet': {'Rows': data[1:]}}] + return type('Paginator', (), {'paginate': lambda self, **kwargs: iter(pages)})() + + +@pytest.fixture +def client(): + fake = FakeAthenaClient() + with patch('cdx_toolkit.indexes.backends.athena.make_client', return_value=fake), \ + patch('cdx_toolkit.indexes.backends.athena.time.sleep'): + yield fake + + +def test_rows_and_cost(client, caplog): + with caplog.at_level(logging.INFO): + rows = list(AthenaClient('ccindex', 's3://b/p/').iter_rows('SELECT 1')) + assert rows == [dict(zip(COLUMNS, row)) for row in ROWS] + assert client.started[0]['QueryExecutionContext'] == {'Database': 'ccindex'} + assert client.started[0]['ResultConfiguration'] == {'OutputLocation': 's3://b/p/'} + assert 'Athena scanned 2.00 GB, estimated cost ~$0.0100' in caplog.text + + +def test_index_athena_end_to_end(client, capsys): + main(['index', ATHENA, '--hostname', 'example.com', '--yes']) + + records = [json.loads(line) for line in capsys.readouterr().out.splitlines()] + assert records[0] == { + 'url': 'https://example.com/', 'timestamp': '20240713102233', 'status': '200', + 'filename': 'crawl-data/a.warc.gz', 'offset': '10', 'length': '100', + } + assert len(records) == 2 + assert 'FROM ccindex' in client.started[0]['QueryString'] + assert client.started[0]['QueryExecutionContext'] == {'Database': 'mydb'} + + +def test_athena_count(): + fake = FakeAthenaClient(rows=[['2']]) + fake.get_paginator = lambda name: type('P', (), {'paginate': lambda self, **kw: iter([ + {'ResultSet': {'Rows': [{'Data': [{'VarCharValue': 'n'}]}, {'Data': [{'VarCharValue': '42'}]}]}} + ])})() + with patch('cdx_toolkit.indexes.backends.athena.make_client', return_value=fake), \ + patch('cdx_toolkit.indexes.backends.athena.time.sleep'): + assert AthenaBackend(ATHENA, assume_yes=True).count(IndexQuery(domains=['example.com'])) == 42 + assert fake.started[0]['QueryString'].startswith('SELECT count(*) AS n FROM (') + + +def test_failed_query(caplog): + fake = FakeAthenaClient(states=['FAILED'], reason='SYNTAX_ERROR: line 1:8') + with caplog.at_level(logging.INFO), pytest.raises(Exception, match='Query failed with status: FAILED'): + AthenaClient('ccindex', 's3://b/p/', client=fake).run('SELECT') + assert 'SYNTAX_ERROR' in caplog.text + assert fake.stopped == [] + + +def test_query_timeout_is_cancelled(): + fake = FakeAthenaClient(states=['RUNNING']) + with patch('cdx_toolkit.indexes.backends.athena.time.sleep'), pytest.raises(TimeoutError): + AthenaClient('ccindex', 's3://b/p/', client=fake, max_wait_time=0).run('SELECT 1') + assert fake.stopped == ['qid'] + + +def test_cost_report_without_statistics(caplog): + with caplog.at_level(logging.INFO): + AthenaClient('ccindex', 's3://b/p/', client=FakeAthenaClient(scanned=None)).report_cost('qid') + assert 'Athena scanned' not in caplog.text + + +def test_make_client_uses_the_region(monkeypatch): + pytest.importorskip('boto3') + monkeypatch.delenv('AWS_MAX_ATTEMPTS', raising=False) + monkeypatch.delenv('AWS_RETRY_MODE', raising=False) + monkeypatch.setattr('cdx_toolkit.aws.user_aws_config', lambda: {}) + with patch('boto3.client') as boto3_client: + make_client('eu-west-1') + args, kwargs = boto3_client.call_args + assert args == ('athena',) and kwargs['config'].region_name == 'eu-west-1' + # Common Crawl's recommended retry settings (cdx_toolkit.aws) + assert kwargs['config'].retries == {'mode': 'adaptive', 'total_max_attempts': 100000} + + +class _FakeAthenaClient: + """Minimal stand-in for a boto3 Athena client (no AWS, no network).""" + + def __init__(self, raise_on_wait=None): + self._raise_on_wait = raise_on_wait + self.started = [] + self.stopped = [] + + def start_query_execution(self, **kwargs): + self.started.append(kwargs) + return {'QueryExecutionId': 'qid-123'} + + def get_query_execution(self, QueryExecutionId): + if self._raise_on_wait is not None: + raise self._raise_on_wait + return {'QueryExecution': {'Status': {'State': 'SUCCEEDED'}, + 'Statistics': {'DataScannedInBytes': 1000}}} + + def stop_query_execution(self, QueryExecutionId): + self.stopped.append(QueryExecutionId) + + +def test_run_logs_sql(caplog): + import logging + client = _FakeAthenaClient() + with caplog.at_level(logging.INFO): + AthenaClient('ccindex', 's3://b/out/', client=client).run('SELECT my_special_column FROM ccindex') + assert 'SELECT my_special_column FROM ccindex' in caplog.text + + +def test_run_success_no_cancel(): + client = _FakeAthenaClient() + qid = AthenaClient('ccindex', 's3://b/out/', client=client).run('SELECT 1') + assert qid == 'qid-123' + assert client.stopped == [] + + +def test_run_cancels_on_interrupt(): + client = _FakeAthenaClient(raise_on_wait=KeyboardInterrupt()) + with pytest.raises(KeyboardInterrupt): + AthenaClient('ccindex', 's3://b/out/', client=client).run('SELECT 1') + assert client.stopped == ['qid-123'] + + +def test_run_cancels_on_timeout(): + client = _FakeAthenaClient(raise_on_wait=TimeoutError('too slow')) + with pytest.raises(TimeoutError): + AthenaClient('ccindex', 's3://b/out/', client=client).run('SELECT 1') + assert client.stopped == ['qid-123'] diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py new file mode 100644 index 0000000..9e19e75 --- /dev/null +++ b/tests/unit/test_aws.py @@ -0,0 +1,268 @@ +"""Common Crawl's S3 defaults (cdx_toolkit.aws): used for the settings the user did not configure.""" +from unittest.mock import patch + +import pytest + +from cdx_toolkit import aws +from tests.conftest import requires_aioboto3 + +FULL_DEFAULTS = { + 'region_name': 'us-east-1', + 'retries': {'mode': 'adaptive', 'total_max_attempts': 100000}, + 's3': {'addressing_style': 'virtual', 'payload_signing_enabled': False}, +} + + +@pytest.fixture +def no_user_config(monkeypatch, tmp_path): + for name in ('AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_RETRY_MODE', 'AWS_MAX_ATTEMPTS', 'CDXT_AWS_DEFAULTS', + 'AWS_PROFILE'): + monkeypatch.delenv(name, raising=False) + # botocore reads the AWS config file itself: use an empty one, not the one of the machine + monkeypatch.setenv('AWS_CONFIG_FILE', str(tmp_path / 'aws-config')) + with patch.object(aws, 'user_aws_config', return_value={}): + yield + + +def test_reads_the_aws_config_file(monkeypatch, tmp_path): + pytest.importorskip('botocore') + config_file = tmp_path / 'aws-config' + config_file.write_text( + '[default]\nmax_attempts = 5\ns3 =\n addressing_style = path\n max_concurrent_requests = 4\n' + ) + for name in ('AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_RETRY_MODE', 'AWS_MAX_ATTEMPTS', 'AWS_PROFILE'): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv('AWS_CONFIG_FILE', str(config_file)) + aws.user_aws_config.cache_clear() + try: + assert aws.botocore_config_kwargs() == { + 'region_name': 'us-east-1', + 'retries': {'mode': 'adaptive'}, + 's3': {'payload_signing_enabled': False}, + } + assert aws.max_concurrent_requests() == 4 + finally: + aws.user_aws_config.cache_clear() + + +def test_defaults(no_user_config): + assert aws.botocore_config_kwargs() == FULL_DEFAULTS + assert aws.max_concurrent_requests() == 2 + + +def test_environment_variables_win(no_user_config, monkeypatch): + monkeypatch.setenv('AWS_DEFAULT_REGION', 'eu-west-1') + monkeypatch.setenv('AWS_MAX_ATTEMPTS', '3') + assert aws.botocore_config_kwargs() == { + 'retries': {'mode': 'adaptive'}, + 's3': {'addressing_style': 'virtual', 'payload_signing_enabled': False}, + } + + +def test_aws_config_file_wins(no_user_config): + user_config = { + 'retry_mode': 'standard', 'max_attempts': '10', + 's3': {'addressing_style': 'path', 'max_concurrent_requests': '8'}, + } + with patch.object(aws, 'user_aws_config', return_value=user_config): + assert aws.botocore_config_kwargs() == {'region_name': 'us-east-1', 's3': {'payload_signing_enabled': False}} + assert aws.max_concurrent_requests() == 8 + + +def test_defaults_can_be_disabled(no_user_config, monkeypatch): + monkeypatch.setenv('CDXT_AWS_DEFAULTS', '0') + assert aws.botocore_config_kwargs() == {} + + +def test_botocore_resolves_the_defaults_and_the_user_settings(no_user_config, monkeypatch): + boto3 = pytest.importorskip('boto3') + + def client(**config): + # a new session reads the environment and the (empty) AWS config file + session = boto3.session.Session(aws_access_key_id='x', aws_secret_access_key='x') + return session.client('s3', config=aws.botocore_config(**config)) + + s3 = client() + assert s3.meta.region_name == 'us-east-1' + assert s3.meta.config.retries == {'mode': 'adaptive', 'total_max_attempts': 100000} + assert s3.meta.config.s3 == {'addressing_style': 'virtual', 'payload_signing_enabled': False} + + # a value from the environment is not overridden: botocore reads it itself + monkeypatch.setenv('AWS_MAX_ATTEMPTS', '3') + assert client().meta.config.retries == {'mode': 'adaptive', 'total_max_attempts': 3} + + # an explicit region wins + assert client(region_name='eu-central-1').meta.region_name == 'eu-central-1' + + +def test_configure_fsspec_s3_keeps_fsspec_settings(no_user_config, monkeypatch): + import fsspec.config + + user_conf = {'s3': {'config_kwargs': {'retries': {'mode': 'standard'}}, 'anon': False}} + monkeypatch.setattr(fsspec.config, 'conf', user_conf) + aws.configure_fsspec_s3() + assert fsspec.config.conf['s3'] == { + 'anon': False, + 'max_concurrency': 2, + 'config_kwargs': { + 'region_name': 'us-east-1', + 'retries': {'mode': 'standard', 'total_max_attempts': 100000}, + 's3': {'addressing_style': 'virtual', 'payload_signing_enabled': False}, + }, + } + + monkeypatch.setattr(fsspec.config, 'conf', {}) + monkeypatch.setenv('CDXT_AWS_DEFAULTS', 'off') + aws.configure_fsspec_s3() + assert fsspec.config.conf == {} + + +def test_s3fs_uses_the_defaults(no_user_config, monkeypatch): + s3fs = pytest.importorskip('s3fs') + import fsspec.config + + monkeypatch.setattr(fsspec.config, 'conf', {}) + aws.configure_fsspec_s3() + fs = s3fs.S3FileSystem(skip_instance_cache=True, key='x', secret='x') + assert fs.config_kwargs['retries'] == {'mode': 'adaptive', 'total_max_attempts': 100000} + assert fs.max_concurrency == 2 + + +@requires_aioboto3 +def test_warc_fetcher_s3_clients(no_user_config): + from cdx_toolkit.repackage.jobs import RecordsSource + from cdx_toolkit.repackage.repackager import Repackager + + repackager = Repackager(source=RecordsSource([], None), prefix_path='out', writer_info={}, n_parallel_readers=4) + read_config, write_config = repackager.s3_read_config(), repackager.s3_write_config() + for config in (read_config, write_config): + assert config.region_name == 'us-east-1' + assert config.retries == {'mode': 'adaptive', 'total_max_attempts': 100000} + assert config.s3 == {'addressing_style': 'virtual', 'payload_signing_enabled': False} + assert read_config.max_pool_connections == 12 and read_config.read_timeout == 300 + + +def test_cli_configures_fsspec(no_user_config, monkeypatch, tmp_path): + import fsspec.config + from cdx_toolkit.cli import main + + monkeypatch.setattr(fsspec.config, 'conf', {}) + captures = tmp_path / 'captures.jsonl' + captures.write_text('') + main(['repackage', str(captures), '--prefix', str(tmp_path / 'out')]) + assert fsspec.config.conf['s3']['config_kwargs']['retries']['total_max_attempts'] == 100000 + + +def test_duckdb_settings(no_user_config): + assert aws.duckdb_settings() == [ + "SET s3_region='us-east-1';", + "SET s3_url_style='vhost';", + 'SET http_retries=99999;', + 'SET http_retry_wait_ms=1000;', + 'SET http_retry_backoff=1;', + ] + assert aws.duckdb_settings(region_name='eu-west-1')[0] == "SET s3_region='eu-west-1';" + + +def test_duckdb_settings_use_the_user_configuration(no_user_config, monkeypatch): + monkeypatch.setenv('AWS_REGION', 'eu-central-1') + monkeypatch.setenv('AWS_MAX_ATTEMPTS', '5') + with patch.object(aws, 'user_aws_config', return_value={'s3': {'addressing_style': 'path'}}): + assert aws.duckdb_settings()[:3] == [ + "SET s3_region='eu-central-1';", "SET s3_url_style='path';", 'SET http_retries=4;', + ] + assert aws.aws_region() == 'eu-central-1' + + monkeypatch.setenv('CDXT_AWS_DEFAULTS', '0') + assert aws.duckdb_settings() == ["SET s3_region='eu-central-1';"] # only what the user configured + monkeypatch.delenv('AWS_REGION') + assert aws.duckdb_settings() == [] + + +def test_invalid_region(no_user_config, monkeypatch): + monkeypatch.setenv('AWS_REGION', "us-east-1'; DROP") + with pytest.raises(ValueError, match='invalid AWS region'): + aws.aws_region() + with pytest.raises(ValueError, match='invalid AWS region'): + aws.duckdb_settings() + + +def s3_client(config=None): + """A real botocore S3 client (no requests are sent), to check the configuration botocore resolves.""" + import botocore.session + + session = botocore.session.Session() + return session.create_client('s3', config=config, aws_access_key_id='x', aws_secret_access_key='x') + + +@pytest.mark.parametrize('env, region', [ + ({}, 'us-east-1'), + ({'AWS_REGION': 'eu-west-1'}, 'eu-west-1'), + ({'AWS_DEFAULT_REGION': 'eu-central-1'}, 'eu-central-1'), + ({'AWS_REGION': 'eu-west-1', 'AWS_DEFAULT_REGION': 'eu-central-1'}, 'eu-west-1'), + ({'AWS_REGION': 'eu-west-1', 'CDXT_AWS_DEFAULTS': '0'}, 'eu-west-1'), +]) +def test_effective_region(no_user_config, monkeypatch, env, region): + pytest.importorskip('botocore') + for name, value in env.items(): + monkeypatch.setenv(name, value) + assert s3_client(aws.botocore_config()).meta.region_name == region + # an explicit region wins + assert s3_client(aws.botocore_config(region_name='ap-south-1')).meta.region_name == 'ap-south-1' + + +def test_effective_region_of_the_profile(no_user_config, monkeypatch, tmp_path): + pytest.importorskip('botocore') + (tmp_path / 'aws-config').write_text('[default]\nregion = sa-east-1\n') + with patch.object(aws, 'user_aws_config', return_value={'region': 'sa-east-1'}): + assert s3_client(aws.botocore_config()).meta.region_name == 'sa-east-1' + + +@pytest.mark.parametrize('retries, expected', [ + ({'max_attempts': 3}, {'mode': 'adaptive', 'total_max_attempts': 4}), # botocore: max_attempts = retries + ({'total_max_attempts': 3}, {'mode': 'adaptive', 'total_max_attempts': 3}), + ({'mode': 'standard'}, {'mode': 'standard', 'total_max_attempts': 100000}), +]) +def test_effective_explicit_retries(no_user_config, retries, expected): + pytest.importorskip('botocore') + assert s3_client(aws.botocore_config(retries=retries)).meta.config.retries == expected + + +@pytest.mark.parametrize('retries, expected', [ + ({'max_attempts': 3}, {'mode': 'adaptive', 'max_attempts': 3}), + ({'total_max_attempts': 3}, {'mode': 'adaptive', 'total_max_attempts': 3}), +]) +def test_fsspec_retries_of_the_user_win(no_user_config, monkeypatch, retries, expected): + import fsspec.config + + monkeypatch.setattr(fsspec.config, 'conf', {'s3': {'config_kwargs': {'retries': retries}}}) + aws.configure_fsspec_s3() + assert fsspec.config.conf['s3']['config_kwargs']['retries'] == expected + + +def fsspec_s3_conf(): + import fsspec.config + + return fsspec.config.conf.get('s3', {}) + + +@pytest.mark.parametrize('defaults', ['on', 'off']) +def test_spawned_processes_use_the_defaults(no_user_config, monkeypatch, defaults): + """A spawned process does not inherit the configuration of this one: process_pool() configures it.""" + import multiprocessing + from concurrent.futures import ProcessPoolExecutor + + monkeypatch.setenv('CDXT_AWS_DEFAULTS', defaults) + monkeypatch.setenv('FSSPEC_CONFIG_DIR', '/nonexistent') # not the fsspec configuration of the machine + ctx = multiprocessing.get_context('spawn') + with aws.process_pool(1, mp_context=ctx) as ex: + conf = ex.submit(fsspec_s3_conf).result() + if defaults == 'on': + assert conf['config_kwargs']['retries'] == FULL_DEFAULTS['retries'] + assert conf['config_kwargs']['s3'] == FULL_DEFAULTS['s3'] + assert conf['max_concurrency'] == 2 + else: + assert 'config_kwargs' not in conf + + with ProcessPoolExecutor(1, mp_context=ctx) as ex: # without process_pool() + assert 'config_kwargs' not in ex.submit(fsspec_s3_conf).result() diff --git a/tests/unit/test_cc.py b/tests/unit/test_cc.py index b93f39d..47f1ac7 100644 --- a/tests/unit/test_cc.py +++ b/tests/unit/test_cc.py @@ -47,7 +47,7 @@ def test_apply_cc_defaults(): [{'crawl': 'foo'}, {}], [{'closest': '20180101'}, {'from_ts': '20171003000000', 'to': '20180401000000'}], [{'closest': '20180101', 'to': '20181201'}, {'from_ts': '20171003000000'}], - [{'to': '20180101'}, {'from_ts': '20170131235959'}], + [{'to': '20180101'}, {'from_ts': '20170101235959'}], # one year before the end of 2018-01-01 [{}, {'from_ts': '20170429003859'}], # uses now [{'from_ts': '20100101', 'closest': '20150301'}, {'to': '20150530000000'}], [{'from_ts': '20100101'}, {}], diff --git a/tests/unit/test_cdx_files.py b/tests/unit/test_cdx_files.py new file mode 100644 index 0000000..3b927b4 --- /dev/null +++ b/tests/unit/test_cdx_files.py @@ -0,0 +1,358 @@ +import gzip +import json +import os +import shutil + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.indexes import IndexQuery, UsageError +from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend, load_url_patterns +from cdx_toolkit.indexes.patterns import domain_entry_to_surts, url_pattern_to_surts +from cdx_toolkit.record_io import write_records + +from tests.conftest import TEST_DATA_PATH, requires_aws_s3 + + +CDX_PATH = TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz' + +with gzip.open(CDX_PATH, 'rt') as _f: + LINES = _f.read().splitlines() + + +def count_lines(prefix, day=None): + return sum(1 for line in LINES if line.startswith(prefix) and (day is None or line.split(' ')[1].startswith(day))) + + +@pytest.mark.parametrize( + 'pattern, prefixes, exact', + [ + ('example.com/*', ['com,example)/'], []), + ('example.com/path*', ['com,example)/path'], []), + ('example.com/path/*', ['com,example)/path/'], []), + ('example.com/path?*', ['com,example)/path?'], []), + ('example.com:8080/*', ['com,example:8080)/'], []), + ('*.example.com', ['com,example)', 'com,example,', 'com,example:'], []), + ('https://www.example.com/a', [], ['com,example)/a']), + ], +) +def test_url_pattern_to_surts(pattern, prefixes, exact): + assert url_pattern_to_surts(pattern) == (prefixes, exact) + + +@pytest.mark.parametrize('pattern', ['ex*ample.com', '*.example.com/path', '*.']) +def test_invalid_url_patterns(pattern): + with pytest.raises(UsageError): + url_pattern_to_surts(pattern) + + +def test_select_by_hostname_domain_and_url(): + backend = CdxFilesBackend(str(CDX_PATH)) + + assert backend.count(IndexQuery(hostnames=['biodiv.mnhn.fr'])) == count_lines('fr,mnhn,biodiv)') + assert backend.count(IndexQuery(domains=['missiondefrance.fr'])) == count_lines('fr,missiondefrance,') + assert backend.count(IndexQuery(domains=['mnhn.fr'], urls=['*.missiondefrance.fr'])) == len(LINES) + assert backend.count(IndexQuery(urls=['biodiv.mnhn.fr/fr/taxonomy*'])) == count_lines('fr,mnhn,biodiv)/fr/taxonomy') + + exact_key = LINES[0].split(' ')[0] + records = list(backend.records(IndexQuery(urls=[json.loads(LINES[0].split(' ', 2)[2])['url']]))) + assert {r['urlkey'] for r in records} == {exact_key} + + +def test_time_range_fgrep_and_limit(): + backend = CdxFilesBackend(str(CDX_PATH)) + query = IndexQuery(domains=['missiondefrance.fr'], from_ts='20240716', to='20240716') + assert backend.count(query) == count_lines('fr,missiondefrance,', day='20240716') + + records = list(backend.records(IndexQuery(domains=['mnhn.fr'], url_fgrep='/taxonomy/', limit=5))) + assert 0 < len(records) <= 5 + assert all('/taxonomy/' in r['url'] and r['urlkey'].startswith('fr,mnhn,biodiv)') for r in records) + assert set(records[0]) >= {'urlkey', 'timestamp', 'url', 'filename', 'offset', 'length'} + + +def test_limit_applies_before_the_url_filters(tmp_path, capsys): + path = tmp_path / 'index.cdx' + path.write_text(''.join( + f'com,example)/{name} 20240101000000 {json.dumps({"url": f"https://example.com/{name}"})}\n' + for name in ('drop', 'keep') + )) + backend = CdxFilesBackend(str(path)) + query = IndexQuery(domains=['example.com'], limit=1, url_fgrep='keep') + assert list(backend.records(query)) == [] + query.url_fgrep, query.url_fgrepv = None, 'drop' + assert list(backend.records(query)) == [] + assert backend.count(IndexQuery(domains=['example.com'], limit=1)) == 1 + assert backend.count(IndexQuery(domains=['example.com'])) == 2 + + +def test_url_file_formats(tmp_path): + urls = ['*.missiondefrance.fr', 'biodiv.mnhn.fr/fr/taxonomy*'] + txt = tmp_path / 'urls.txt' + txt.write_text('# target URLs\n' + '\n'.join(urls) + '\n\n') + assert load_url_patterns(str(txt)) == urls + + write_records(({'name': str(i), 'target': u} for i, u in enumerate(urls)), str(tmp_path / 'urls.csv'), 'csv') + assert load_url_patterns(str(tmp_path / 'urls.csv'), 'target') == urls + with pytest.raises(UsageError, match="no column 'url'"): + load_url_patterns(str(tmp_path / 'urls.csv')) + with pytest.raises(UsageError, match='--url-column needs'): + load_url_patterns(str(txt), 'target') + + backend = CdxFilesBackend(str(CDX_PATH)) + expected = count_lines('fr,missiondefrance,') + count_lines('fr,mnhn,biodiv)/fr/taxonomy') + assert backend.count(IndexQuery(url_file=str(txt))) == expected + assert backend.count(IndexQuery(url_file=str(tmp_path / 'urls.csv'), url_column='target')) == expected + + +def test_url_file_parquet(tmp_path): + pytest.importorskip('pyarrow') + write_records(iter([{'url': '*.missiondefrance.fr'}]), str(tmp_path / 'urls.parquet'), 'parquet') + assert load_url_patterns(str(tmp_path / 'urls.parquet')) == ['*.missiondefrance.fr'] + + +@pytest.mark.parametrize('processes', [1, 2]) +def test_one_output_per_cdx_file(tmp_path, processes): + for name in ('a/cdx-00001.gz', 'b/cdx-00002.gz', 'b/cdx-00003.gz'): + (tmp_path / 'in' / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(CDX_PATH, tmp_path / 'in' / name) + # no match in this file: no output + with gzip.open(tmp_path / 'in' / 'b' / 'cdx-00003.gz', 'wt') as f: + f.write(next(line for line in LINES if line.startswith('fr,missiondefrance')) + '\n') + out = tmp_path / 'out' + + main([ + 'index', str(tmp_path / 'in' / '*' / 'cdx-*.gz'), '--hostname', 'biodiv.mnhn.fr', '-o', str(out), + '--processes', str(processes), + ]) + + outputs = sorted(p.relative_to(out).as_posix() for p in out.rglob('*') if p.is_file()) + assert outputs == ['a/cdx-00001.gz', 'b/cdx-00002.gz'] + with gzip.open(out / 'a' / 'cdx-00001.gz', 'rt') as f: + # the lines are copied unchanged + assert f.read().splitlines() == [line for line in LINES if line.startswith('fr,mnhn,biodiv)')] + + +def test_cli_combined_output_and_errors(tmp_path, capsys): + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '--limit', '3', '--fields', 'url', '--output-format', 'csv']) + out = capsys.readouterr().out.splitlines() + assert out[0] == 'url' and len(out) == 4 + + for args, error in [ + ([], 'select captures from the CDX files'), + (['--domain', 'mnhn.fr', '--filter', '=status:200'], '--filter cannot be used with the CDX files index'), + (['--domain', 'mnhn.fr', '--processes', '2'], '--processes can only be used'), + (['--domain', 'mnhn.fr', '-o', str(tmp_path / 'out'), '--limit', '1'], '--limit cannot be used'), + (['--domain', 'mnhn.fr', '-o', str(tmp_path / 'out'), '--fields', 'url'], '--fields cannot be used'), + (['--url-column', 'x', '--domain', 'mnhn.fr'], '--url-column and --url-file-type need --url-file'), + (['--domain', 'mnhn.fr', '--limit', '0'], '--limit must be at least 1'), + (['--domain', 'mnhn.fr', '--limit', '-1'], '--limit must be at least 1'), + (['--domain', 'mnhn.fr', '--count', '--url-fgrep', 'x'], '--count cannot be used with --url-fgrep'), + ]: + with pytest.raises(SystemExit): + main(['index', str(CDX_PATH)] + args) + assert error in capsys.readouterr().err, args + + +@pytest.mark.parametrize('index', ['cc', 'cc-url-index-s3']) +def test_url_file_only_for_cdx_files(capsys, index): + with pytest.raises(SystemExit): + main(['index', index, '--url-file', 'urls.txt']) + err = capsys.readouterr().err + assert '--url-file cannot be used with the' in err + if index != 'cc': + assert 'JOIN' in err + + +def test_dry_run(capsys): + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '--url', 'example.com/a', '--dry-run']) + out = capsys.readouterr().out + assert '1 CDX file(s)' in out and 'prefix fr,mnhn)' in out and 'exact com,example)/a' in out + + +@pytest.mark.parametrize( + 'entry, prefixes', + [ + ('example.com', ['com,example)', 'com,example,', 'com,example:']), + ('https://example.com/', ['com,example)', 'com,example,', 'com,example:']), + ('example.com/reports', ['com,example)/reports']), + ], +) +def test_domain_entry_to_surts(entry, prefixes): + assert domain_entry_to_surts(entry) == prefixes + + +def test_url_file_type_domain(): + """With --url-file-type domain, host entries match their domain with subdomains (like #73's filter_cdx).""" + url_list = str(TEST_DATA_PATH / 'repackage' / 'url_list.txt') + backend = CdxFilesBackend(str(CDX_PATH)) + + # e.g. 'bibliotheque.missiondefrance.fr' is a single URL as a pattern ... + assert backend.count(IndexQuery(url_file=url_list)) == 0 + # ... and a domain with --url-file-type domain; 'biodiv.mnhn.fr/fr/taxonomy' is a URL prefix + expected = count_lines('fr,missiondefrance,bibliotheque)') + count_lines('fr,mnhn,biodiv)/fr/taxonomy') + assert backend.count(IndexQuery(url_file=url_list, url_file_type='domain')) == expected == len(LINES) + + +@pytest.mark.parametrize( + 'args, error', + [ + (['--domain', 'mnhn.fr', '--url-file-type', 'domain'], '--url-column and --url-file-type need --url-file'), + (['--url-file', 'urls.txt', '--url-file-type', 'regex'], "invalid choice: 'regex'"), + ], +) +def test_url_file_type_errors(capsys, args, error): + with pytest.raises(SystemExit): + main(['index', str(CDX_PATH)] + args) + assert error in capsys.readouterr().err + + +def test_no_globs_over_http(capsys): + for command in (['index', 'https://example.org/indexes/cdx-*.gz', '--domain', 'mnhn.fr'], + ['repackage', 'https://example.org/indexes/cdx-*.gz']): + with pytest.raises(SystemExit): + main(command) + assert 'globs do not work with http(s) URLs' in capsys.readouterr().err + + +def test_one_output_per_cdx_file_with_url_filters(tmp_path): + out = tmp_path / 'out' + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '--url-fgrep', '/taxonomy/', '--url-fgrepv', '/term/1', + '-o', str(out)]) + with gzip.open(out / CDX_PATH.name, 'rt') as f: + lines = f.read().splitlines() + + def url(line): + return json.loads(line.split(' ', 2)[2])['url'] + + expected = [line for line in LINES if line.startswith('fr,mnhn,') and '/taxonomy/' in url(line)] + assert lines == [line for line in expected if '/term/1' not in url(line)] + assert lines + + +@pytest.mark.parametrize('output, inputs', [ + ('in/a.cdx.gz', ['in/a.cdx.gz']), + ('./in/../in/a.cdx.gz', ['in/a.cdx.gz']), + ('hard.cdx.gz', ['in/a.cdx.gz']), + ('link.cdx.gz', ['in/a.cdx.gz']), + ('in/a.cdx.gz', ['in/*.cdx.gz']), + ('in', ['in/*.cdx.gz']), + ('urls.txt', ['in/a.cdx.gz', '--url-file', 'urls.txt']), +]) +def test_output_must_not_overwrite_the_input(tmp_path, monkeypatch, capsys, output, inputs): + monkeypatch.chdir(tmp_path) + (tmp_path / 'in').mkdir() + shutil.copy(CDX_PATH, tmp_path / 'in' / 'a.cdx.gz') + (tmp_path / 'urls.txt').write_text('*.mnhn.fr\n') + os.link(tmp_path / 'in' / 'a.cdx.gz', tmp_path / 'hard.cdx.gz') + try: + os.symlink(tmp_path / 'in' / 'a.cdx.gz', tmp_path / 'link.cdx.gz') + except OSError: # Windows without the privilege + if output == 'link.cdx.gz': + pytest.skip('no symlinks') + before = {p: p.read_bytes() for p in tmp_path.rglob('*') if p.is_file()} + + with pytest.raises(SystemExit) as e: + main(['index'] + inputs + ['--domain', 'mnhn.fr', '-o', output, '--output-format', 'cdxj']) + assert e.value.code == 2 + assert 'would overwrite the input' in capsys.readouterr().err + assert {p: p.read_bytes() for p in tmp_path.rglob('*') if p.is_file()} == before + + +@pytest.mark.parametrize('processes', [1, 2]) +def test_one_output_per_cdx_file_needs_an_empty_directory(tmp_path, capsys, processes): + out = tmp_path / 'out' + out.mkdir() # empty: fine + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '-o', str(out), '--processes', str(processes)]) + output = out / CDX_PATH.name + first = output.read_bytes() + assert first + + # a second run, even one that selects nothing, would leave the outputs of the first run in place + with pytest.raises(SystemExit) as e: + main(['index', str(CDX_PATH), '--domain', 'example.com', '-o', str(out), '--processes', str(processes)]) + assert e.value.code == 2 + assert 'is not empty' in capsys.readouterr().err + assert output.read_bytes() == first + + +@pytest.mark.parametrize('output, per_file', [ + ('out', True), ('out/', True), ('out.v2/', True), ('.out', True), + ('out.cdxj', False), ('out.jsonl.gz', False), +]) +def test_one_output_per_cdx_file_only_without_file_extension(tmp_path, monkeypatch, output, per_file): + monkeypatch.chdir(tmp_path) + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '-o', output]) + assert (tmp_path / output).is_dir() == per_file + + +@pytest.mark.parametrize('output', ['out.txt', 'out.gz', 'out.v2']) +def test_unknown_output_extension(tmp_path, monkeypatch, capsys, output): + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit) as e: + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '-o', output]) + assert e.value.code == 2 + assert 'cannot detect the output format' in capsys.readouterr().err + assert not (tmp_path / output).exists() + + main(['index', str(CDX_PATH), '--domain', 'mnhn.fr', '-o', output, '--output-format', 'jsonl']) + assert (tmp_path / output).is_file() + + +@requires_aws_s3 +def test_output_checks_on_s3(s3_tmpdir, capsys): + import fsspec + + fs = fsspec.filesystem('s3') + fs.put_file(str(CDX_PATH), f'{s3_tmpdir}/in/a.cdx.gz') + fs.pipe_file(f'{s3_tmpdir}/out/unrelated.txt', b'x') + for output, error in [ + (f'{s3_tmpdir}/in/a.cdx.gz', 'would overwrite the input'), + (f'{s3_tmpdir}/in', 'would overwrite the input'), + (f'{s3_tmpdir}/out', 'is not empty'), + ]: + with pytest.raises(SystemExit): + main(['index', f'{s3_tmpdir}/in/*.cdx.gz', '--domain', 'mnhn.fr', '-o', output]) + assert error in capsys.readouterr().err, output + assert fs.cat_file(f'{s3_tmpdir}/in/a.cdx.gz') == CDX_PATH.read_bytes() + + main(['index', f'{s3_tmpdir}/in/*.cdx.gz', '--domain', 'mnhn.fr', '-o', f'{s3_tmpdir}/out2']) + assert fs.exists(f'{s3_tmpdir}/out2/a.cdx.gz') + + +@pytest.mark.parametrize('timestamp, valid', [ + ('2024', True), ('202402', True), ('20240229', True), ('2024022923', True), ('202402292359', True), + ('20240229235959', True), ('2024080123595', True), + ('202400', False), ('202413', False), ('20230229', False), ('20240230', False), ('20240231', False), + ('2024022924', False), ('202402292360', False), ('20245', False), ('2024-01', False), ('202', False), + ('202402292359590', False), ('２０２４', False), +]) +def test_timestamps_are_validated_by_every_backend(capsys, timestamp, valid): + """Invalid calendar dates are rejected (exit 2, no traceback) before a query, by CDX files, SQL and the API.""" + for index in (str(CDX_PATH), 'cc-url-index-s3', 'cc'): + for option in ('--from', '--to'): + args = ['index', index, '--domain', 'example.com', option, timestamp, '--dry-run'] + if valid: + main(args) + capsys.readouterr() + else: + with pytest.raises(SystemExit) as e: + main(args) + assert e.value.code == 2 + err = capsys.readouterr().err + assert f'{option}: invalid timestamp' in err and 'Traceback' not in err, (index, option) + + +def test_cdx_files_and_sql_select_the_same_time_range(tmp_path, capsys): + """Leap day and end of month: the bounds of the file backend (string comparison) and SQL are the same.""" + pytest.importorskip('duckdb') + from cdx_toolkit.indexes.backends.sql_builder import sql_timestamp + + assert sql_timestamp('202402') == "TIMESTAMP '2024-02-01 00:00:00'" + assert sql_timestamp('202402', end=True) == "TIMESTAMP '2024-03-01 00:00:00'" + lines = [f'com,example)/ {ts} {json.dumps({"url": "https://example.com/", "offset": ts})}\n' + for ts in ('20240131235959', '20240201000000', '20240229235959', '20240301000000')] + path = tmp_path / 'index.cdx' + path.write_text(''.join(lines)) + main(['index', str(path), '--domain', 'example.com', '--from', '202402', '--to', '202402', '--fields', 'offset', + '--output-format', 'csv']) + assert capsys.readouterr().out.split() == ['offset', '20240201000000', '20240229235959'] diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index d8edc08..7be322f 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,8 +1,18 @@ -from argparse import Namespace +from argparse import ArgumentTypeError +import json from unittest.mock import Mock, patch -import cdx_toolkit.cli +import pytest + +from warcio.archiveiterator import ArchiveIterator + +import cdx_toolkit.cli.repackage +import cdx_toolkit.repackage.warc_records import cdx_toolkit.warc +from cdx_toolkit.cli import main +from cdx_toolkit.cli.common import parse_size +from cdx_toolkit.repackage.jobs import RepackageResult +import cdx_toolkit.cli.index def _make_fake_record(): @@ -19,80 +29,275 @@ def _make_fake_record(): ) -def _build_warc_cmd(tmp_path, **overrides): - cmd = Namespace( - prefix=str(tmp_path / 'cli'), - subprefix=None, - url='http://example.com', - url_fgrep=None, - url_fgrepv=None, - creator=None, - operator=None, - ) - for key, value in overrides.items(): - setattr(cmd, key, value) - return cmd +RECORD = cdx_toolkit.repackage.warc_records.get_bytes_from_warc_record(_make_fake_record(), gzip=True) + + +def _range_response(offset=10): + """The response to a range request for RECORD at `offset` of a WARC file.""" + content_range = f'bytes {offset}-{offset + len(RECORD) - 1}/*' + return Mock(status_code=206, headers={'Content-Range': content_range}, content=RECORD) + + +def _write_captures(path, captures): + path.write_text(''.join(json.dumps(c) + '\n' for c in captures)) + return str(path) -def _make_capture(is_revisit=False, url='http://example.com'): - capture = Mock() - capture.__getitem__ = lambda self, key: {'url': url, 'timestamp': '20240101120000'}[key] - capture.is_revisit.return_value = is_revisit - capture.fetch_warc_record.return_value = _make_fake_record() - return capture +IA_CAPTURE = {'url': 'http://example.com', 'timestamp': '20240101120000', 'status': '200', 'mime': 'text/html'} +CC_CAPTURE = { + 'url': 'http://example.com', 'timestamp': '20240101120000', 'status': '200', + 'filename': 'crawl-data/x.warc.gz', 'offset': '10', 'length': str(len(RECORD)), +} -def test_warcer_logs_revisit(tmp_path, caplog): - """warcer should log a warning when iterating over a revisit capture.""" - cmd = _build_warc_cmd(tmp_path) - fake_cdx = Mock() - fake_cdx.iter.return_value = iter([_make_capture(is_revisit=True)]) +def test_repackage_with_wb_logs_revisit(tmp_path, caplog): + captures = _write_captures(tmp_path / 'captures.jsonl', [dict(IA_CAPTURE, mime='warc/revisit')]) + prefix = str(tmp_path / 'out') - with patch.object(cdx_toolkit.cli, 'setup_cdx_fetcher_and_kwargs', return_value=(fake_cdx, {})): - with caplog.at_level('WARNING', logger='cdx_toolkit.cli'): - cdx_toolkit.cli.warcer(cmd, cmdline='test') + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc', return_value=_make_fake_record()) as fetch: + with caplog.at_level('WARNING', logger='cdx_toolkit.repackage.readers.wayback'): + main(['repackage', captures, '--wb', 'ia', '--prefix', prefix]) + assert fetch.call_args.kwargs['wb'] == 'https://web.archive.org/web' assert 'revisit record being resolved' in caplog.text - assert list(tmp_path.glob('cli-*.warc.gz')) + assert [p.name for p in tmp_path.glob('out-*')] == ['out-001.warc.gz'] + + +def test_repackage_fetches_by_warc_location(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + resp = _range_response() + + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=resp) as get: + main(['repackage', captures, '--prefix', str(tmp_path / 'out'), '--warc-download-prefix', 'https://x.org/']) + + assert get.call_args.args == ('https://x.org/crawl-data/x.warc.gz',) + assert get.call_args.kwargs['headers'] == {'Range': f'bytes=10-{9 + len(RECORD)}'} + + with open(tmp_path / 'out-001.warc.gz', 'rb') as f: + records = list(ArchiveIterator(f)) + assert [r.rec_type for r in records] == ['warcinfo', 'response'] + + +@pytest.mark.parametrize('existing, wb', [ + ('out-000000.extracted.warc.gz', True), + ('out-001.warc.gz', True), + ('out-000000.extracted.warc.gz', False), +]) +def test_repackage_rejects_existing_warc_files_with_and_without_wb(tmp_path, capsys, existing, wb): + """The same rule for both writers: no overwriting, no adding to the files of an earlier run.""" + (tmp_path / existing).write_bytes(b'earlier run') + captures = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE if wb else CC_CAPTURE]) + wb_args = ['--wb', 'ia'] if wb else [] + + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc') as fetch, \ + patch('cdx_toolkit.repackage.readers.http.myrequests_get') as get, \ + pytest.raises(SystemExit) as exc: + main(['repackage', captures, '--prefix', str(tmp_path / 'out')] + wb_args) + + assert exc.value.code == 2 + assert 'exist already' in capsys.readouterr().err + fetch.assert_not_called() + get.assert_not_called() + assert sorted(p.name for p in tmp_path.glob('out-*')) == [existing] + assert (tmp_path / existing).read_bytes() == b'earlier run' + + +def test_repackage_without_warc_location_or_wb(tmp_path, capsys): + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE, IA_CAPTURE]) + + with pytest.raises(SystemExit) as e: + main(['repackage', captures, '--prefix', str(tmp_path / 'out')]) + + assert e.value.code == 2 + assert 'use --wb' in capsys.readouterr().err + assert not list(tmp_path.glob('out*')) # nothing is written + + +def test_repackage_without_captures_writes_no_file(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', []) + + main(['repackage', captures, '--prefix', str(tmp_path / 'out')]) + + assert not list(tmp_path.glob('out*')) + + +def test_repackage_default_prefix(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=_range_response()): + main(['repackage', captures]) + + assert [p.name for p in tmp_path.glob('*.warc.gz')] == ['REPACKAGE-001.warc.gz'] + + +def test_index_to_parquet_and_repackage(tmp_path, capsys): + pytest.importorskip('pyarrow') + captures = tmp_path / 'captures.parquet' + + with patch.object(cdx_toolkit.cli.index, 'open_index') as open_index: + open_index.return_value.records.return_value = iter([CC_CAPTURE]) + main(['index', 'cc', '--url', 'example.com/*', '-o', str(captures)]) + with pytest.raises(SystemExit): + main(['index', 'cc', '--url', 'example.com/*', '--output-format', 'parquet']) + assert 'cannot be written to stdout' in capsys.readouterr().err + + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=_range_response()): + main(['repackage', str(captures), '--prefix', str(tmp_path / 'out')]) + with open(tmp_path / 'out-001.warc.gz', 'rb') as f: + assert [r.rec_type for r in ArchiveIterator(f)] == ['warcinfo', 'response'] + + +def test_repackage_options_are_passed_to_the_warc_fetcher(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', []) + + with patch('cdx_toolkit.repackage.repackager.Repackager') as repackager: + repackager.return_value.run.return_value = RepackageResult() + main([ + 'repackage', captures, '--prefix', 'FOO', '--subprefix', 'BAR', '--size', '1MiB', + '--creator', 'me', '--description', 'test', '--add-metadata-record', 'a.txt', + '--parallel-readers', '4', '--warc-download-prefix', 's3://commoncrawl', + ]) + + kwargs = repackager.call_args.kwargs + assert kwargs['prefix_path'] == 'FOO' + assert kwargs['writer_subprefix'] == 'BAR' + assert kwargs['max_file_size'] == 1024 * 1024 + assert kwargs['write_paths_as_metadata_records'] == ['a.txt'] + assert kwargs['n_parallel_readers'] == 4 + assert [job.url for job in kwargs['source'].iter_range_jobs()] == [] + info = kwargs['writer_info'] + assert info['isPartOf'] == 'FOO-BAR' + assert info['creator'] == 'me' + assert info['description'] == 'test' + + +def test_repackage_with_processes_uses_multiprocess(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + + result = RepackageResult(1, 1) + with patch('cdx_toolkit.repackage.multiprocess.run_multiprocess_repackage', return_value=result) as run: + main(['repackage', captures, '--prefix', str(tmp_path / 'out'), '--processes', '3', '--subprefix', 'SUB']) + + kwargs = run.call_args.kwargs + assert (kwargs['n_processes'], kwargs['writer_subprefix']) == (3, 'SUB') + assert [job.filename for job in kwargs['source'].iter_range_jobs()] == ['crawl-data/x.warc.gz'] + + +def test_repackage_with_wb_rejects_fetcher_options(tmp_path, capsys): + captures = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE]) + + with pytest.raises(SystemExit): + main(['repackage', captures, '--wb', 'ia', '--processes', '2']) + assert '--processes cannot be used with --wb' in capsys.readouterr().err + + +def test_repackage_with_wb_passes_size_and_warcinfo_to_the_repackager(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE]) + + with patch('cdx_toolkit.repackage.repackager.Repackager') as repackager: + repackager.return_value.run.return_value = RepackageResult() + main([ + 'repackage', captures, '--wb', 'ia', '--prefix', 'FOO', '--subprefix', 'BAR', '--size', '1MiB', + '--creator', 'me', '--description', 'test', '--parallel-readers', '2', + ]) + + kwargs = repackager.call_args.kwargs + assert (kwargs['prefix_path'], kwargs['writer_subprefix']) == ('FOO', 'BAR') + assert (kwargs['max_file_size'], kwargs['n_parallel_readers']) == (1024 * 1024, 2) + info = kwargs['writer_info'] + assert (info['isPartOf'], info['creator'], info['description']) == ('FOO-BAR', 'me', 'test') + jobs = list(kwargs['source'].iter_range_jobs()) + assert [job.url for job in jobs] == ['https://web.archive.org/web/20240101120000id_/http%3A//example.com'] + + +def test_repackage_with_wb_rejects_common_crawl_captures(tmp_path, capsys): + """cdxt index cc ... | cdxt repackage --wb ia would replay the captures from another archive.""" + captures = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc') as fetch, pytest.raises(SystemExit) as exc: + main(['repackage', captures, '--wb', 'ia', '--prefix', str(tmp_path / 'out')]) + + assert exc.value.code == 2 + assert 'is a Common Crawl capture' in capsys.readouterr().err + fetch.assert_not_called() + assert list(tmp_path.glob('out-*')) == [] + + +def test_repackage_with_wb_writes_the_warc_series(tmp_path): + """Replays are written like copied records: -001.warc.gz with the warcinfo and metadata records, also with + parallel readers.""" + captures = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE] * 3) + metadata = tmp_path / 'metadata.txt' + metadata.write_text('about this run') + + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc', side_effect=lambda *a, **k: _make_fake_record()): + main(['repackage', captures, '--wb', 'ia', '--prefix', str(tmp_path / 'out'), '--parallel-readers', '2', + '--add-metadata-record', str(metadata)]) + + assert [p.name for p in tmp_path.glob('out-*')] == ['out-001.warc.gz'] + with open(tmp_path / 'out-001.warc.gz', 'rb') as f: + assert [r.rec_type for r in ArchiveIterator(f)] == ['warcinfo', 'metadata'] + ['response'] * 3 + + +@pytest.mark.parametrize( + 'value, expected', + [('1000', 1000), ('1kB', 1000), ('100MB', 100 * 1000 ** 2), ('1.5GB', 1500 * 1000 ** 2), ('1GiB', 1024 ** 3)], +) +def test_parse_size(value, expected): + assert parse_size(value) == expected + + +@pytest.mark.parametrize('value', ['', 'GB', '1XB', '-1']) +def test_parse_size_invalid(value): + with pytest.raises(ArgumentTypeError): + parse_size(value) -def test_warcer_skips_fgrep_and_fgrepv(tmp_path, caplog): - """warcer should skip URLs not matching --url-fgrep or matching --url-fgrepv.""" - cmd = _build_warc_cmd(tmp_path, url_fgrep='wanted', url_fgrepv='forbidden') - captures = [ - _make_capture(url='http://example.com/nope'), # fails fgrep - _make_capture(url='http://example.com/wanted/forbidden'), # fails fgrepv - _make_capture(url='http://example.com/wanted/keep'), # kept - ] - fake_cdx = Mock() - fake_cdx.iter.return_value = iter(captures) +def test_verbose_before_and_after_subcommand(tmp_path): + captures = _write_captures(tmp_path / 'captures.jsonl', []) + with patch.object(cdx_toolkit.cli, 'set_loglevel') as set_loglevel: + main(['-v', 'repackage', captures, '--prefix', str(tmp_path / 'out')]) + assert set_loglevel.call_args.args == (1,) + main(['repackage', '-vv', captures, '--prefix', str(tmp_path / 'out')]) + assert set_loglevel.call_args.args == (2,) - with patch.object(cdx_toolkit.cli, 'setup_cdx_fetcher_and_kwargs', return_value=(fake_cdx, {})): - with caplog.at_level('DEBUG', logger='cdx_toolkit.cli'): - cdx_toolkit.cli.warcer(cmd, cmdline='test') - assert 'not warcing due to fgrep' in caplog.text - assert 'not warcing due to fgrepv' in caplog.text - assert captures[0].fetch_warc_record.call_count == 0 - assert captures[1].fetch_warc_record.call_count == 0 - assert captures[2].fetch_warc_record.call_count == 1 +@pytest.mark.parametrize('n_failing', [1, 2]) +def test_repackage_exits_1_if_captures_could_not_be_fetched(tmp_path, capsys, n_failing): + captures = [dict(CC_CAPTURE, offset=str(i * len(RECORD))) for i in range(2)] + path = _write_captures(tmp_path / 'captures.jsonl', captures) + calls = [] + def get(url, headers=None, **kwargs): + calls.append(url) + if len(calls) <= n_failing: + raise ConnectionError('network down') + return _range_response(offset=int(headers['Range'][len('bytes='):].split('-')[0])) -def test_warcer_passes_size_to_writer(tmp_path): - """warcer should pop 'size' out of kwargs and forward it to the writer.""" - cmd = _build_warc_cmd(tmp_path) - fake_cdx = Mock() - fake_cdx.iter.return_value = iter([]) - captured = {} + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=get), \ + pytest.raises(SystemExit) as exc: + main(['repackage', path, '--prefix', str(tmp_path / 'out')]) + assert exc.value.code == 1 + assert f'{n_failing} of 2 captures could not be fetched' in capsys.readouterr().err + assert len(list(tmp_path.glob('out-*.warc.gz'))) == (1 if n_failing == 1 else 0) - real_get_writer = cdx_toolkit.warc.get_writer - def spy_get_writer(prefix, subprefix, info, **kwargs): - captured.update(kwargs) - return real_get_writer(prefix, subprefix, info, **kwargs) +def test_repackage_fails_if_the_output_cannot_be_created(tmp_path): + path = _write_captures(tmp_path / 'captures.jsonl', [CC_CAPTURE]) + response = _range_response() + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', return_value=response), \ + patch('cdx_toolkit.repackage.writers.local.aiofiles.open', side_effect=PermissionError('read-only')), \ + pytest.raises(PermissionError): + main(['repackage', path, '--prefix', str(tmp_path / 'out')]) + assert list(tmp_path.glob('out-*')) == [] - with patch.object(cdx_toolkit.cli, 'setup_cdx_fetcher_and_kwargs', return_value=(fake_cdx, {'size': 42})): - with patch.object(cdx_toolkit.warc, 'get_writer', side_effect=spy_get_writer): - cdx_toolkit.cli.warcer(cmd, cmdline='test') - assert captured == {'size': 42} +def test_repackage_with_wb_counts_failed_captures(tmp_path, capsys): + path = _write_captures(tmp_path / 'captures.jsonl', [IA_CAPTURE, IA_CAPTURE]) + fetched = [RuntimeError('404'), _make_fake_record()] + with patch('cdx_toolkit.repackage.readers.wayback.fetch_wb_warc', side_effect=fetched), \ + pytest.raises(SystemExit) as exc: + main(['repackage', path, '--wb', 'ia', '--prefix', str(tmp_path / 'out')]) + assert exc.value.code == 1 + assert '1 of 2 captures could not be fetched' in capsys.readouterr().err diff --git a/tests/unit/test_cli_legacy.py b/tests/unit/test_cli_legacy.py new file mode 100644 index 0000000..fb2397f --- /dev/null +++ b/tests/unit/test_cli_legacy.py @@ -0,0 +1,286 @@ +'''The command lines of cdx_toolkit 0.9.39 (cdx_toolkit/cli/legacy.py), without requests.''' +import json +import shlex +from unittest.mock import Mock, patch + +import pytest + +import cdx_toolkit.cli.legacy as legacy +from cdx_toolkit.cli import build_parser, main +from cdx_toolkit.cli.legacy import build_legacy_parser, is_legacy_command_line, suggested_replacement +from cdx_toolkit.repackage.jobs import RepackageResult + + +CAPTURES = [ + {'url': 'https://example.com/', 'status': '200', 'timestamp': '20240101000000', 'digest': 'A', 'mime': 'text/html'}, + {'url': 'https://example.com/b', 'status': '404', 'timestamp': '20240102000000', 'digest': 'B'}, +] + + +def parse(cmdline: str): + return build_legacy_parser().parse_args(shlex.split(cmdline)) + + +@pytest.mark.parametrize( + 'args, expected', + [ + ('--cc --limit 10 iter example.com/*', True), + ('--crawl 2 iter example.com', True), + ('--limit=5 iter example.com', True), + ('-v -v --cc size example.com', True), + ('--ia warc example.com', True), + ('iter cc --url example.com', True), # the unreleased alias: rejected by the old grammar + ('--limit iter example.com', True), # old options before the command: strict old grammar (error) + ('-- iter example.com', True), + ('index cc --url example.com', False), + ('-v index cc --url iter', False), + ('-vv repackage captures.jsonl', False), + ('index cc --crawl iter --url example.com', False), + ('--version', False), + ('-h', False), + ('', False), + ], +) +def test_is_legacy_command_line(args, expected): + assert is_legacy_command_line(args.split()) == expected + + +def test_a_value_of_an_old_option_is_not_the_command(): + # `--source iter`: iter is the value of --source, the command is size + assert is_legacy_command_line(['--source', 'iter', 'size', 'x']) + ns = parse('--source https://example.com/cdx --filter iter size x') + assert (ns.cmd, ns.filter) == ('size', ['iter']) + + +@pytest.mark.parametrize( + 'args, message', + [ + ('--cc iter', 'the following arguments are required: url'), + ('--cc iter --url example.com', 'unrecognized arguments: --url'), + ('iter cc --url example.com', 'unrecognized arguments: --url'), + ('--limit iter example.com', "invalid int value: 'iter'"), + ('--cc --limit', 'expected one argument'), + ('--cc iter --size 3 example.com', 'unrecognized arguments: --size'), + ('--limit 3 iter example.com', 'must specify --cc, --ia, or a --source'), + ('--source example.com iter example.com', 'use the http(s) URL of a CDX API server'), + ('--cc iter --all-fields --csv example.com', '--all-fields cannot be used with --csv'), + ('--cc --wb https://example.com/wb warc example.com', 'cannot specify --wb for Common Crawl'), + ('--source https://example.com/cdx warc example.com', 'needs --wb (a wayback machine) or'), + ('--ia --wb ia warc --warc-download-prefix https://example.com example.com', 'cannot specify both'), + ], +) +def test_invalid_old_command_lines_exit_2_without_requests(capsys, args, message): + with patch.object(legacy, 'open_index') as open_index, patch.object(legacy, 'fetcher') as fetcher, \ + patch('cdx_toolkit.repackage.repackage') as repackage: + with pytest.raises(SystemExit) as e: + main(args.split()) + assert e.value.code == 2 + assert message in capsys.readouterr().err + open_index.assert_not_called() + fetcher.assert_not_called() + repackage.assert_not_called() + + +@pytest.mark.parametrize( + 'args, source', + [ + ('--crawl 1 --ia iter x', 'cc'), + ('--cc --ia iter x', 'cc'), + ('--source https://example.com/cdx --ia iter x', 'ia'), + ('--source https://example.com/cdx iter x', 'https://example.com/cdx'), + ], +) +def test_source_precedence(args, source): + assert legacy.legacy_source(parse(args)) == source + + +def run_iter(capsys, args, captures=CAPTURES): + """Run an old iter command line on `captures` from the CDX API backend.""" + backend = Mock() + backend.records.return_value = iter(captures) + with patch.object(legacy, 'open_index', return_value=backend) as open_index: + main(args.split()) + out, err = capsys.readouterr() + return out, err, open_index, backend + + +def test_iter_text_output(capsys): + out, err, open_index, backend = run_iter( + capsys, '--cc --crawl CC-MAIN-2024-33 --limit 2 --from 2024 --filter =status:200 iter example.com/*' + ) + assert out.splitlines() == [ + 'status 200, timestamp 20240101000000, url https://example.com/', + 'status 404, timestamp 20240102000000, url https://example.com/b', + ] + assert open_index.call_args.args == ('cc', 'cdx-api') + query = backend.records.call_args.args[0] + assert (query.urls, query.crawl, query.limit, query.from_ts, query.filters) == ( + ['example.com/*'], 'CC-MAIN-2024-33', 2, '2024', ['=status:200']) + assert err.count('\n') == 1 + assert 'the old syntax, which is deprecated and will be removed in a future version' in err + assert ("suggested replacement: cdxt index cc --url 'example.com/*' --crawl CC-MAIN-2024-33 --limit 2 " + '--from 2024 --filter =status:200 --output-format csv --fields status,timestamp,url') in err + + +def test_iter_fields_are_deduplicated_and_missing_ones_omitted(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --fields mime,url,mime x') + assert out.splitlines() == ['mime text/html, url https://example.com/', 'url https://example.com/b'] + + +def test_iter_all_fields(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --fields url --all-fields x') + assert out.splitlines()[0] == ( + 'digest A, mime text/html, status 200, timestamp 20240101000000, url https://example.com/' + ) + + +def test_iter_jsonl(capsys): + out, err, _, _ = run_iter(capsys, '--ia iter --jsonl x') + assert out.splitlines()[0] == '{"status": "200", "timestamp": "20240101000000", "url": "https://example.com/"}' + assert "cdxt index ia --url x --fields status,timestamp,url (see" in err + + +def test_iter_jsonl_all_fields(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --jsonl --all-fields x') + assert json.loads(out.splitlines()[1]) == CAPTURES[1] + + +def test_iter_csv(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --csv --fields url,digest,mime x') + assert out == 'digest,mime,url\r\nA,text/html,https://example.com/\r\nB,,https://example.com/b\r\n' + + +def test_iter_csv_header_without_captures(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --csv x', captures=[]) + assert out == 'status,timestamp,url\r\n' + + +def test_iter_jsonl_and_csv(capsys): + # as in 0.9.39: the CSV header, then JSON lines + out, _, _, _ = run_iter(capsys, '--ia iter --csv --jsonl x') + assert out.splitlines()[:2] == [ + 'status,timestamp,url', '{"status": "200", "timestamp": "20240101000000", "url": "https://example.com/"}' + ] + + +@pytest.mark.parametrize( + 'args, method, kwargs', + [ + ('--cc --get iter x', 'get', {}), # CDXFetcher.get() adds limit=1000 + ('--cc --get --limit 3 --closest 2017 iter x', 'get', {'limit': 3, 'closest': '2017'}), + ('--cc --closest 2017 --to 2018 iter x', 'iter', {'closest': '2017', 'to': '2018'}), # paged, as before + ], +) +def test_iter_get_and_closest_use_the_fetcher(capsys, args, method, kwargs): + cdx = Mock() + getattr(cdx, method).return_value = CAPTURES[:1] + with patch.object(legacy, 'fetcher', return_value=cdx), patch.object(legacy, 'open_index') as open_index: + main(args.split()) + getattr(cdx, method).assert_called_once_with('x', **kwargs) + open_index.assert_not_called() + out, err = capsys.readouterr() + assert out == 'status 200, timestamp 20240101000000, url https://example.com/\n' + if args == '--cc --get iter x': + assert 'cdxt index cc --url x --limit 1000 --output-format csv' in err + + +def test_fetcher_options(): + with patch('cdx_toolkit.CDXFetcher') as fetcher: + legacy.fetcher(parse('--crawl 2 --cc-mirror https://m/ --cc-sort ascending iter x'), 'cc') + assert fetcher.call_args.kwargs == { + 'source': 'cc', 'crawl': ['2'], 'cc_mirror': 'https://m/', 'cc_sort': 'ascending'} + legacy.fetcher(parse('--ia --cc-mirror https://m/ iter x'), 'ia') # ignored for other sources, as before + assert fetcher.call_args.kwargs == {'source': 'ia'} + + +def test_size_is_not_capped_by_the_limit(capsys): + cdx = Mock() + cdx.get_size_estimate.return_value = 1500 + with patch.object(legacy, 'fetcher', return_value=cdx): + main('--cc --limit 10 size --details x'.split()) + cdx.get_size_estimate.assert_called_once_with('x', limit=10, details=True) + out, err = capsys.readouterr() + assert out == '1500\n' + assert "suggested replacement: cdxt index cc --url x --limit 10 --count" in err + + +def run_warc(args, result=RepackageResult(2, 2, 0), captures=CAPTURES): + with patch.object(legacy, 'legacy_captures', return_value=iter(captures)) as legacy_captures, \ + patch('cdx_toolkit.repackage.repackage', return_value=result) as repackage: + main(args.split()) + return legacy_captures, repackage + + +def test_warc_streams_into_repackage(capsys): + legacy_captures, repackage = run_warc( + '--cc --get --limit 2 warc --subprefix SUB --size 1000 --creator me --url-fgrepv /b x') + assert legacy_captures.call_args.kwargs == {'use_get': False} # 0.9.39 iterated with --get too + args, kwargs = repackage.call_args + assert list(args[0]) == CAPTURES[:1] # --url-fgrepv after the limit + assert args[1] == 'TEST' + assert kwargs['sort'] is False + assert kwargs['warc_download_prefix'] == 'https://data.commoncrawl.org' and 'wb' not in kwargs + assert (kwargs['subprefix'], kwargs['size'], kwargs['creator'], kwargs['operator']) == ('SUB', 1000, 'me', None) + assert kwargs['description'] == ( + 'warc extraction generated with: cdxt --cc --get --limit 2 warc --subprefix SUB --size 1000 --creator me ' + '--url-fgrepv /b x' + ) + assert ('suggested replacement: cdxt index cc --url x --limit 2 --url-fgrepv /b | cdxt repackage --prefix TEST ' + '--subprefix SUB --size 1000 --creator me --no-sort') in capsys.readouterr().err + + +@pytest.mark.parametrize( + 'args, content, suggestion', + [ + ('--ia warc x', {'wb': 'ia'}, '| cdxt repackage --prefix TEST --wb ia --no-sort'), + ('--ia warc --warc-download-prefix https://w x', {'warc_download_prefix': 'https://w'}, + '--warc-download-prefix https://w --no-sort'), + ('--source https://e/cdx --wb https://e/wb warc x', {'wb': 'https://e/wb'}, '--wb https://e/wb --no-sort'), + ('--crawl 1 warc --warc-download-prefix s3://commoncrawl x', {'warc_download_prefix': 's3://commoncrawl'}, + '--warc-download-prefix s3://commoncrawl --no-sort'), + ], +) +def test_warc_content_source(capsys, args, content, suggestion): + _, repackage = run_warc(args) + kwargs = repackage.call_args.kwargs + assert {k: kwargs[k] for k in ('wb', 'warc_download_prefix') if k in kwargs} == content + assert suggestion in capsys.readouterr().err + + +def test_warc_exits_1_if_captures_failed(capsys): + with pytest.raises(SystemExit) as e: + run_warc('--cc warc x', result=RepackageResult(2, 1, 1)) + assert e.value.code == 1 + assert '1 of 2 captures could not be fetched' in capsys.readouterr().err + + +@pytest.mark.parametrize( + 'args', + [ + '--cc --limit 10 iter example.com/*', + "--crawl 'CC-MAIN-2024-33,CC-MAIN-2024-30' --cc-sort ascending --from 2024 --to 20240410 " + "--filter '!=status:200' --filter =mime:text/html iter --fields url,digest example.com/*", + '--ia --get --closest 2017 iter --jsonl --all-fields example.com', + '--source https://example.com/cdx iter --csv example.com', + '--cc --limit 3 size --details example.com', + "--ia warc --prefix 'my dir/IA' --subprefix X --size 100 --creator 'a b' --operator c --url-fgrep a " + '--url-fgrepv robots.txt example.com/*', + '--cc --cc-mirror https://index.commoncrawl.org/ warc --warc-download-prefix s3://commoncrawl example.com', + ], +) +def test_suggested_replacement_is_a_valid_new_command_line(args): + suggestion = suggested_replacement(parse(args)) + for part in suggestion.split(' | '): + tokens = shlex.split(part) + assert tokens[0] == 'cdxt' + build_parser().parse_args(tokens[1:]) # exits on an error + + +def test_legacy_captures_without_a_crawl_or_limit(): + backend = Mock() + with patch.object(legacy, 'open_index', return_value=backend) as open_index: + legacy.legacy_captures(parse('--source https://example.com/cdx.php iter x'), 'https://example.com/cdx.php', + use_get=False) + assert open_index.call_args.args == ('https://example.com/cdx.php', 'cdx-api') # not detected from the URL + query = backend.records.call_args.args[0] + assert (query.crawl, query.limit, query.filters) == (None, None, []) diff --git a/tests/unit/test_confirm_cost.py b/tests/unit/test_confirm_cost.py new file mode 100644 index 0000000..8eb46bf --- /dev/null +++ b/tests/unit/test_confirm_cost.py @@ -0,0 +1,55 @@ +from unittest.mock import patch + +import pytest + +from cdx_toolkit.indexes.backends.sql import confirm_cost + + +def est(n_crawls): + return n_crawls, 'athena' + + +def test_small_crawl_set_no_prompt(): + with patch('builtins.input') as inp: + confirm_cost(*est(5), confirmed=False) + inp.assert_not_called() + + +def test_confirmed_flag_bypasses_prompt(): + with patch('builtins.input') as inp: + confirm_cost(*est(None), confirmed=True) + inp.assert_not_called() + + +def test_large_crawl_set_non_tty_aborts(): + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin: + stdin.isatty.return_value = False + with pytest.raises(SystemExit): + confirm_cost(*est(11), confirmed=False) + + +def test_unknown_crawls_non_tty_aborts(): + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin: + stdin.isatty.return_value = False + with pytest.raises(SystemExit): + confirm_cost(*est(None), confirmed=False) + + +def test_tty_yes_proceeds(): + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin, patch('builtins.input', return_value='y'): + stdin.isatty.return_value = True + confirm_cost(*est(None), confirmed=False) # should not raise + + +def test_tty_empty_answer_aborts(): + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin, patch('builtins.input', return_value=''): + stdin.isatty.return_value = True + with pytest.raises(SystemExit): + confirm_cost(*est(None), confirmed=False) + + +def test_tty_no_answer_aborts(): + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin, patch('builtins.input', return_value='n'): + stdin.isatty.return_value = True + with pytest.raises(SystemExit): + confirm_cost(*est(11), confirmed=False) diff --git a/tests/unit/test_duckdb.py b/tests/unit/test_duckdb.py new file mode 100644 index 0000000..569f9b9 --- /dev/null +++ b/tests/unit/test_duckdb.py @@ -0,0 +1,112 @@ +"""DuckDB queries of a local parquet file, and the connection setup with a fake connection (no network).""" +from unittest.mock import MagicMock, patch + +import pytest + +from cdx_toolkit.indexes.backends import duckdb as duckdb_source +from cdx_toolkit.indexes.backends.duckdb import build_from_clause, iter_duckdb_rows + +from tests.conftest import requires_duckdb + + +def test_from_clause(): + assert build_from_clause('s3://b/index/', ['CC-MAIN-2024-33']) == ( + "read_parquet(['s3://b/index/crawl=CC-MAIN-2024-33/subset=warc/*.parquet'], hive_partitioning=true)" + ) + assert build_from_clause('s3://b/index', None) == ( + "read_parquet(['s3://b/index/crawl=*/subset=warc/*.parquet'], hive_partitioning=true)" + ) + + +@requires_duckdb +def test_rows_from_parquet(tmp_path): + import duckdb + + path = tmp_path / 'index.parquet' + duckdb.connect().execute(f''' + COPY (SELECT * FROM (VALUES ('crawl-data/a.warc.gz', 10), ('crawl-data/b.warc.gz', 20)) + AS t(warc_filename, warc_record_offset)) + TO '{path}' (FORMAT parquet) + ''') + assert list(iter_duckdb_rows(f"SELECT * FROM read_parquet('{path}')")) == [ + {'warc_filename': 'crawl-data/a.warc.gz', 'warc_record_offset': 10}, + {'warc_filename': 'crawl-data/b.warc.gz', 'warc_record_offset': 20}, + ] + + +def test_without_duckdb(): + with patch.object(duckdb_source, '_HAS_DUCKDB', False), pytest.raises(RuntimeError, match=r'cdx_toolkit\[duckdb\]'): + list(iter_duckdb_rows('SELECT 1')) + + +def run_with_fake_connection(query): + """The rows of the query, the executed statements and the connection.""" + con = MagicMock() + con.execute.return_value.description = [('n',)] + con.execute.return_value.fetchmany.side_effect = [[(1,)], []] + fake_duckdb = MagicMock(connect=MagicMock(return_value=con)) + with patch.object(duckdb_source, '_HAS_DUCKDB', True), \ + patch.object(duckdb_source, 'duckdb', fake_duckdb, create=True): + rows = list(iter_duckdb_rows(query)) + return rows, [c.args[0] for c in con.execute.call_args_list], con + + +@pytest.mark.parametrize('query, httpfs, s3', [ + ("SELECT 1 AS n FROM read_parquet('/data/index/*.parquet')", False, False), + ("SELECT 1 AS n FROM read_parquet('https://data.commoncrawl.org/x.parquet')", True, False), + ("SELECT 1 AS n FROM read_parquet('s3://commoncrawl/x.parquet')", True, True), +]) +def test_extensions_and_credentials_only_when_needed(monkeypatch, query, httpfs, s3): + monkeypatch.setattr('cdx_toolkit.aws.user_aws_config', lambda: {}) + rows, statements, _ = run_with_fake_connection(query) + assert rows == [{'n': 1}] + assert ('INSTALL httpfs; LOAD httpfs;' in statements) == httpfs + assert ('SET http_timeout=120;' in statements) == httpfs + assert ('INSTALL aws; LOAD aws;' in statements) == s3 + assert any(stmt.startswith('SET s3_region=') for stmt in statements) == s3 + assert statements[-1] == query + + +def test_connection_setup_and_hf_filesystem(monkeypatch): + for name in ('AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_MAX_ATTEMPTS', 'CDXT_AWS_DEFAULTS'): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr('cdx_toolkit.aws.user_aws_config', lambda: {}) + con = MagicMock() + con.execute.return_value.description = [('n',)] + con.execute.return_value.fetchmany.side_effect = [[(1,)], []] + fake_duckdb = MagicMock(connect=MagicMock(return_value=con)) + hf_fs = MagicMock() + + fake_hf = MagicMock(HfFileSystem=MagicMock(return_value=hf_fs), __version__='1.7.2') + query = "SELECT count(*) AS n FROM read_parquet(['hf://buckets/x/y/*.parquet', 's3://commoncrawl/x.parquet'])" + with patch.object(duckdb_source, '_HAS_DUCKDB', True), \ + patch.object(duckdb_source, 'duckdb', fake_duckdb, create=True), \ + patch.dict('sys.modules', {'huggingface_hub': fake_hf}): + rows = list(iter_duckdb_rows(query)) + + assert rows == [{'n': 1}] + statements = [c.args[0] for c in con.execute.call_args_list] + assert "SET TimeZone='UTC';" in statements + # Common Crawl's recommended S3 settings (cdx_toolkit.aws) and a timeout of 2 minutes + for statement in ("SET s3_region='us-east-1';", "SET s3_url_style='vhost';", 'SET http_retries=99999;', + 'SET http_retry_wait_ms=1000;', 'SET http_retry_backoff=1;', 'SET http_timeout=120;'): + assert statement in statements + assert "CREATE SECRET (TYPE s3, PROVIDER credential_chain, REGION 'us-east-1');" in statements + con.register_filesystem.assert_called_once_with(hf_fs) + con.close.assert_called_once() + + +@pytest.mark.parametrize('output', ['index.parquet', 'idx/index.parquet', 'idx/new.parquet']) +def test_output_must_not_overwrite_the_index(tmp_path, monkeypatch, capsys, output): + from cdx_toolkit.cli import main + + monkeypatch.chdir(tmp_path) + (tmp_path / 'idx').mkdir() + for path in ('index.parquet', 'idx/index.parquet'): + (tmp_path / path).write_bytes(b'PAR1') + location = 'idx/' if output.startswith('idx/') else 'index.parquet' + with pytest.raises(SystemExit) as e: + main(['index', location, '--index-backend', 'duckdb', '--domain', 'example.com', '-o', output]) + assert e.value.code == 2 + assert 'would overwrite the input' in capsys.readouterr().err + assert (tmp_path / 'index.parquet').read_bytes() == b'PAR1' diff --git a/tests/unit/test_hf.py b/tests/unit/test_hf.py new file mode 100644 index 0000000..e947c3b --- /dev/null +++ b/tests/unit/test_hf.py @@ -0,0 +1,40 @@ +import sys +from unittest.mock import patch + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.hf import check_huggingface_hub, uses_hf, version_tuple + + +def test_version_tuple(): + assert version_tuple('1.7.2') == (1, 7, 2) + assert version_tuple('2.0.0rc1') == (2, 0, 0) + assert version_tuple('1.10') == (1, 10) + assert version_tuple('1.7.2') >= (1, 7, 2) > version_tuple('1.7.1') > version_tuple('0.30.0') + + +@pytest.mark.parametrize('version, ok', [('0.30.0', False), ('1.7.1', False), ('1.7.2', True), ('2.0.0', True)]) +def test_check_huggingface_hub(version, ok): + with patch.dict(sys.modules, {'huggingface_hub': type(sys)('huggingface_hub')}): + sys.modules['huggingface_hub'].__version__ = version + if ok: + check_huggingface_hub() + else: + with pytest.raises(RuntimeError, match=r'huggingface_hub >= 1\.7\.2'): + check_huggingface_hub() + + +def test_missing_huggingface_hub(): + with patch.dict(sys.modules, {'huggingface_hub': None}), pytest.raises(RuntimeError, match=r'cdx_toolkit\[hf\]'): + check_huggingface_hub() + + +def test_cli_checks_the_version_first(capsys): + assert uses_hf(['index', 'cc-url-index-hf']) and uses_hf(['repackage', 'x', '--prefix', 'hf://buckets/a/b/c']) + assert not uses_hf(['index', 'cc', '--url', 'example.com/*']) + with patch.dict(sys.modules, {'huggingface_hub': type(sys)('huggingface_hub')}), pytest.raises(SystemExit) as e: + sys.modules['huggingface_hub'].__version__ = '0.30.0' + main(['index', 'cc-url-index-hf', '--domain', 'example.com', '--dry-run']) + assert e.value.code == 2 + assert 'huggingface_hub >= 1.7.2, not 0.30.0' in capsys.readouterr().err diff --git a/tests/unit/test_indexes.py b/tests/unit/test_indexes.py new file mode 100644 index 0000000..5e11413 --- /dev/null +++ b/tests/unit/test_indexes.py @@ -0,0 +1,237 @@ +from unittest.mock import patch + +import pytest + +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index +from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend +from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend +from cdx_toolkit.indexes.resolve import is_cdx_api_location + + +@pytest.mark.parametrize( + 'location, is_api', + [ + ('cc', True), + ('ia', True), + ('http://index.commoncrawl.org', True), + ('https://index.commoncrawl.org/', True), + ('https://index.commoncrawl.org/CC-MAIN-2024-30-index', True), + ('https://web.archive.org/cdx/search/cdx', True), + ('https://data.commoncrawl.org/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-00000.gz', False), + ('https://example.com/index.cdxj', False), + ('https://example.com/indexes/*', False), + ('s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/*.cdx.gz', False), + ('./captures.cdx', False), + ('/data/index', False), + ], +) +def test_is_cdx_api_location(location, is_api): + assert is_cdx_api_location(location) == is_api + + +def test_open_index(): + backend = open_index('cc', cc_mirror='https://mirror.example.com/', cc_sort='ascending') + assert isinstance(backend, CdxApiBackend) + assert (backend.source, backend.cc_mirror, backend.cc_sort) == ('cc', 'https://mirror.example.com/', 'ascending') + + cdx_files = open_index('s3://commoncrawl/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz') + assert isinstance(cdx_files, CdxFilesBackend) + with pytest.raises(UsageError, match='unknown index backend'): + open_index('cc', backend='parquet') + + +def test_query_options_used(): + query = IndexQuery(urls=['example.com/*'], limit=10) + assert query.options_used() == {'urls', 'limit'} + assert IndexQuery().options_used() == set() + + +def test_query_single_values_are_lists(): + query = IndexQuery(urls='example.com/*', domains='example.org', filters='=status:200', languages='eng') + assert (query.urls, query.domains, query.filters, query.languages) == ( + ['example.com/*'], ['example.org'], ['=status:200'], ['eng']) + + +@pytest.mark.parametrize('method', ['records', 'count', 'explain']) +def test_public_methods_validate_the_query(method): + """A Python caller does not need to call validate() first.""" + with patch.object(CdxApiBackend, 'select') as select, patch.object(CdxApiBackend, 'select_count') as count, \ + pytest.raises(UsageError, match='at least one --url'): + getattr(CdxApiBackend('cc'), method)(IndexQuery(crawl='3')) + select.assert_not_called() + count.assert_not_called() + + +def test_count_with_url_fgrep(): + with pytest.raises(UsageError, match='--count cannot be used with --url-fgrep'): + CdxApiBackend('cc').count(IndexQuery(urls='example.com/*', url_fgrep='x')) + + +def test_cdx_api_validate(): + CdxApiBackend('cc').validate(IndexQuery(domains=['example.com'], crawl='3')) + + with pytest.raises(UsageError, match='at least one --url'): + CdxApiBackend('cc').validate(IndexQuery(crawl='3')) + with pytest.raises(UsageError, match='--crawl can only be used'): + CdxApiBackend('ia').validate(IndexQuery(urls=['example.com'], crawl='3')) + + +@pytest.mark.parametrize('expression', ['=status:200', '!=status:200', '~status:2..', '!~status:2..']) +def test_ia_filter_syntax(expression): + """The Internet Archive has no =, ~ filters: a usage error before any request, not a ValueError when the + captures are read.""" + with patch('cdx_toolkit.CDXFetcher') as fetcher, pytest.raises(UsageError, match='ia does not support'): + CdxApiBackend('ia').records(IndexQuery(urls='example.com/*', filters=expression)) + fetcher.assert_not_called() + + CdxApiBackend('ia').validate(IndexQuery(urls='example.com/*', filters=['status:200', '!mime:text/.*'])) + CdxApiBackend('cc').validate(IndexQuery(urls='example.com/*', filters=expression)) + + +def test_unsupported_option(): + class NoFilterBackend(CdxApiBackend): + supported_options = CdxApiBackend.supported_options - {'filters'} + + with pytest.raises(UsageError, match='--filter cannot be used with the CDX API index'): + NoFilterBackend('cc').validate(IndexQuery(urls=['example.com'], filters=['=status:200'])) + + +def _capture(url): + return {'url': url, 'timestamp': '20240101000000'} + + +def test_cdx_api_url_patterns_and_limit(): + query = IndexQuery( + urls=['example.com/*'], hostnames=['www.example.org'], domains=['example.net'], + from_ts='2020', filters=['=status:200'], limit=3, + ) + results = { + 'example.com/*': [_capture('http://example.com/a'), _capture('http://example.com/b')], + 'www.example.org': [_capture('http://www.example.org/'), _capture('http://www.example.org/x')], + 'example.net': [_capture('http://example.net/')], + } + + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.iter.side_effect = lambda url, **params: iter(results[url]) + records = list(CdxApiBackend('cc').records(query)) + + assert [r['url'] for r in records] == ['http://example.com/a', 'http://example.com/b', 'http://www.example.org/'] + + calls = fetcher.return_value.iter.call_args_list + assert len(calls) == 2 # the limit was reached before the domain query + # with several selectors, the limit is applied after removing duplicates, not by the server + assert calls[0].args == ('example.com/*',) + assert calls[0].kwargs == {'from_ts': '2020', 'filter': ['=status:200']} + assert calls[1].args == ('www.example.org',) + assert calls[1].kwargs == {'from_ts': '2020', 'filter': ['=status:200'], 'matchType': 'host'} + + +def test_cdx_api_single_selector_limit_is_sent_to_the_server(): + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.iter.return_value = iter([_capture('http://example.com/a')]) + list(CdxApiBackend('cc').records(IndexQuery(urls=['example.com/*'], limit=3))) + assert fetcher.return_value.iter.call_args.kwargs == {'limit': 3} + + +def test_cdx_api_overlapping_selectors_return_each_capture_once(): + def capture(url, timestamp, offset): + return {'url': url, 'timestamp': timestamp, 'filename': 'a.warc.gz', 'offset': offset} + + shared = capture('http://example.com/a', '20240101000000', '1') + again = capture('http://example.com/a', '20240201000000', '2') # same URL, another capture + other_crawl = capture('http://example.com/a', '20240101000000', '3') # same URL and time, another WARC record + results = { + 'example.com/*': [shared, again], + 'example.com': [dict(shared), other_crawl, capture('http://www.example.com/', '20240101000000', '4')], + } + query = IndexQuery(urls=['example.com/*'], domains=['example.com']) + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.iter.side_effect = lambda url, **params: iter(results[url]) + records = list(CdxApiBackend('cc').records(query)) + assert [r['offset'] for r in records] == ['1', '2', '3', '4'] + + # the limit counts distinct captures + query.limit = 3 + records = list(CdxApiBackend('cc').records(query)) + assert [r['offset'] for r in records] == ['1', '2', '3'] + + +def test_cdx_api_closest_uses_get_and_crawl_is_normalized(): + query = IndexQuery(domains=['example.com'], closest='20170615', crawl='CC-MAIN-2024-33,CC-MAIN-2024-30') + + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.get.return_value = [_capture('http://example.com/')] + records = list(CdxApiBackend('cc').records(query)) + + assert len(records) == 1 + assert fetcher.call_args.kwargs == {'source': 'cc', 'crawl': ['CC-MAIN-2024-33', 'CC-MAIN-2024-30']} + assert fetcher.return_value.get.call_args.kwargs == {'closest': '20170615', 'matchType': 'domain'} + + +def test_cdx_api_count(): + query = IndexQuery(urls=['example.com/*'], domains=['example.org'], limit=10) + + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.get_size_estimate.side_effect = [3000, 1500, 3000, 1500] + assert CdxApiBackend('ia').count(query) == 10 # capped by the limit + calls = fetcher.return_value.get_size_estimate.call_args_list + assert calls[1].args == ('example.org',) + assert calls[1].kwargs == {'limit': 10, 'matchType': 'domain'} + + query.limit = None + assert CdxApiBackend('ia').count(query) == 4500 + + +def test_records_url_fgrep(): + backend = CdxApiBackend('cc') + query = IndexQuery(urls=['example.com/*'], url_fgrep='/keep', url_fgrepv='robots.txt') + captures = [ + _capture('http://example.com/drop'), + _capture('http://example.com/keep/robots.txt'), + _capture('http://example.com/keep/this'), + ] + + with patch.object(backend, 'select', return_value=iter(captures)): + records = list(backend.records(query)) + + assert [r['url'] for r in records] == ['http://example.com/keep/this'] + + +def test_cdx_api_hostname_is_exact(): + """matchType=host selects the host with and without www.: the other host's captures are dropped before the + limit (which is not sent to the server).""" + results = { + 'www.example.com': [ + _capture('http://example.com/'), _capture('https://EXAMPLE.com/a'), + _capture('http://www.example.com/'), _capture('https://www.example.com:8443/b'), + _capture('http://www.example.com/c'), + ], + 'example.org/*': [_capture('http://example.org/')], + } + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.iter.side_effect = lambda url, **params: iter(results[url]) + + records = list(CdxApiBackend('cc').records(IndexQuery(hostnames=['WWW.example.com.'], limit=2))) + assert [r['url'] for r in records] == ['http://www.example.com/', 'https://www.example.com:8443/b'] + assert fetcher.return_value.iter.call_args.kwargs == {'matchType': 'host'} # no server limit + + # combined with another selector: a union + query = IndexQuery(hostnames=['www.example.com'], urls=['example.org/*']) + records = list(CdxApiBackend('cc').records(query)) + assert [r['url'] for r in records] == [ + 'http://example.org/', 'http://www.example.com/', 'https://www.example.com:8443/b', + 'http://www.example.com/c', + ] + + plan = CdxApiBackend('cc').explain(IndexQuery(hostnames=['www.example.com'], limit=2)) + assert plan == ('cc: url=www.example.com matchType=host\n only captures of the host www.example.com\n' + 'limit: 2 (after removing duplicates and other hosts)\n') + + +def test_cdx_api_closest_with_hostname(): + with patch('cdx_toolkit.CDXFetcher') as fetcher: + fetcher.return_value.get.return_value = [_capture('http://example.com/'), _capture('http://www.example.com/')] + query = IndexQuery(hostnames=['www.example.com'], closest='20170615', limit=1) + records = list(CdxApiBackend('cc').records(query)) + assert [r['url'] for r in records] == ['http://www.example.com/'] + assert fetcher.return_value.get.call_args.kwargs == {'closest': '20170615', 'matchType': 'host'} diff --git a/tests/unit/test_myrequests.py b/tests/unit/test_myrequests.py index 588dc4f..d3f3a50 100644 --- a/tests/unit/test_myrequests.py +++ b/tests/unit/test_myrequests.py @@ -1,16 +1,22 @@ import requests +import responses from unittest.mock import patch import pytest from cdx_toolkit.myrequests import myrequests_get +@responses.activate def test_get_without_error(): + responses.add(responses.GET, 'http://example.com/', body='') resp = myrequests_get('http://example.com') assert resp.status_code == 200 -def test_get_with_connection_error(caplog): +def test_get_with_connection_error(caplog, monkeypatch): + from cdx_toolkit import myrequests + + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'example.com', 0) # every attempt takes its turn with patch('requests.get') as mock_get: mock_get.side_effect = requests.exceptions.ConnectionError('Connection failed') @@ -25,3 +31,117 @@ def test_get_with_connection_error(caplog): log_levels = [r.levelname for r in caplog.records] assert log_levels.count('WARNING') == 2 assert log_levels.count('ERROR') == 1 + + +def test_concurrent_requests_keep_the_minimum_interval(): + import threading + + from cdx_toolkit.myrequests import HostRateLimiter + + host = 'interval-test.example.org' + limiter = HostRateLimiter(3.0, {host: 0.5}) + times = [] + threads = [threading.Thread(target=lambda: times.append(limiter.reserve(host, 1000.0))) for _ in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert sorted(times) == [1000.0, 1000.5, 1001.0, 1001.5] + assert limiter.minimum_interval('other.example.org') == 3.0 + + +def test_finish_spaces_the_next_request_from_the_end_of_the_previous_one(): + from cdx_toolkit.myrequests import HostRateLimiter + + limiter = HostRateLimiter(10.0, {}) + with patch('cdx_toolkit.myrequests.time.monotonic', return_value=2000.0): + limiter.finish('example.org') + assert limiter.reserve('example.org', 2001.0) == 2010.0 + + +@pytest.mark.parametrize('host, fatal', [('unknown-host.example.org', True), ('data.commoncrawl.org', False)]) +def test_dns_error_is_only_fatal_for_unknown_hosts(monkeypatch, host, fatal): + from cdx_toolkit import myrequests + + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, host, 0) + error = requests.exceptions.ConnectionError('[Errno 8] nodename nor servname provided, or not known') + with patch('requests.get', side_effect=error) as get, pytest.raises(ValueError) as e: + myrequests_get(f'https://{host}/x', retry_max_sec=0.01, raise_error_after_n_errors=1) + if fatal: + assert 'invalid hostname' in str(e.value) and get.call_count == 1 + else: + assert '2 failures' in str(e.value) and get.call_count == 2 # retried + + +def test_retries_take_their_turn(monkeypatch): + """Three parallel requests to a host, the first answered with a 503: every attempt, also the retry, reserves + its own turn, the minimum interval from the others (the retry used to start in the third request's turn). + The reserved times are checked, not the measured ones, which a slow machine delays.""" + import threading + from unittest.mock import Mock + + from cdx_toolkit import myrequests + + host = 'retry-turn.example.org' + interval = 0.1 + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, host, interval) + reserved = [] + original_reserve = myrequests.RATE_LIMITER.reserve + + def reserve(hostname, now): + fetch_time = original_reserve(hostname, now) + reserved.append(fetch_time) + return fetch_time + + monkeypatch.setattr(myrequests.RATE_LIMITER, 'reserve', reserve) + calls = [] + lock = threading.Lock() + + def get(url, **kwargs): + with lock: + calls.append(url) + first = len(calls) == 1 + return Mock(status_code=503 if first else 200, text='') + + with patch('requests.get', side_effect=get): + threads = [threading.Thread(target=myrequests_get, args=(f'https://{host}/{i}',)) for i in range(3)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(calls) == 4 + assert len(reserved) == 4 # one turn per attempt + reserved.sort() + gaps = [b - a for a, b in zip(reserved, reserved[1:])] + assert min(gaps) >= interval - 1e-9, gaps + + +ABORTED = "Error: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))" + + +@pytest.mark.parametrize('status', [200, 400, 500]) +@responses.activate +def test_cdx_server_connection_error_is_retried(monkeypatch, status): + """The index server could not read its index: its answer is the exception's text, with any status.""" + from cdx_toolkit import myrequests + + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'cdx.example.org', 0) + responses.add(responses.GET, 'https://cdx.example.org/cdx', body=ABORTED, status=status) + responses.add(responses.GET, 'https://cdx.example.org/cdx', body='{"url": "https://example.com/"}\n') + + resp = myrequests_get('https://cdx.example.org/cdx', cdx=True) + + assert resp.status_code == 200 and resp.text.startswith('{') + assert len(responses.calls) == 2 + + +@responses.activate +def test_cdx_400_without_connection_error_is_not_retried(monkeypatch): + from cdx_toolkit import myrequests + + monkeypatch.setitem(myrequests.RATE_LIMITER.intervals, 'cdx.example.org', 0) + responses.add(responses.GET, 'https://cdx.example.org/cdx', body='page too big', status=400) + + assert myrequests_get('https://cdx.example.org/cdx', cdx=True).status_code == 400 + assert len(responses.calls) == 1 diff --git a/tests/unit/test_record_io.py b/tests/unit/test_record_io.py new file mode 100644 index 0000000..d85e338 --- /dev/null +++ b/tests/unit/test_record_io.py @@ -0,0 +1,233 @@ +import io +import json +import os + +import pytest + +from cdx_toolkit.record_io import ( + RecordWriter, format_from_path, is_empty_dir, is_glob, overlaps, read_records, sniff_format, write_records, +) + + +RECORDS = [ + {'urlkey': 'com,example)/', 'timestamp': '20240101000000', 'url': 'http://example.com/', 'status': '200', + 'filename': 'crawl-data/a.warc.gz', 'offset': '10', 'length': '20'}, + {'urlkey': 'com,example)/x', 'timestamp': '20240102000000', 'url': 'http://example.com/x', 'status': '404', + 'filename': 'crawl-data/b.warc.gz', 'offset': '30', 'length': '40'}, +] + + +@pytest.mark.parametrize( + 'path, fmt', + [ + ('-', None), + ('out.jsonl', 'jsonl'), + ('out.json.gz', 'jsonl'), + ('s3://bucket/out.cdxj', 'cdxj'), + ('out.cdx.gz', 'cdxj'), + ('OUT.CSV', 'csv'), + ('out.tsv.gz', 'tsv'), + ('out.txt', None), + ], +) +def test_format_from_path(path, fmt): + assert format_from_path(path) == fmt + + +@pytest.mark.parametrize('fmt', ['jsonl', 'cdxj', 'csv', 'tsv']) +@pytest.mark.parametrize('extension', ['', '.gz']) +def test_round_trip(tmp_path, fmt, extension): + path = str(tmp_path / f'records.{fmt}{extension}') + assert write_records(iter(RECORDS), path, fmt) == 2 + + # the format is detected from the content, not the extension + assert list(read_records(path)) == RECORDS + + +def test_writer_fields(): + fh = io.StringIO() + writer = RecordWriter(fh, 'csv', fields=['url', 'status', 'missing']) + for record in RECORDS: + writer.write(record) + assert fh.getvalue().splitlines() == ['url,status,missing', 'http://example.com/,200,', 'http://example.com/x,404,'] + + fh = io.StringIO() + RecordWriter(fh, 'cdxj', fields=['url']).write(RECORDS[0]) + assert fh.getvalue() == 'com,example)/ 20240101000000 {"url": "http://example.com/"}\n' + + +def test_writer_cdxj_requires_urlkey(): + with pytest.raises(ValueError, match='urlkey'): + RecordWriter(io.StringIO(), 'cdxj').write({'url': 'http://example.com/', 'timestamp': '2024'}) + + +def test_read_columnar_index_names(tmp_path): + path = tmp_path / 'range-jobs.csv' + path.write_text('url,warc_filename,warc_record_offset,warc_record_length\nhttp://example.com/,a.warc.gz,1,2\n') + assert list(read_records(str(path))) == [ + {'url': 'http://example.com/', 'filename': 'a.warc.gz', 'offset': '1', 'length': '2'} + ] + + +def test_read_empty_and_stdin(tmp_path, monkeypatch): + path = tmp_path / 'empty.jsonl' + path.write_text('') + assert list(read_records(str(path))) == [] + + monkeypatch.setattr('sys.stdin', io.StringIO(json.dumps(RECORDS[0]) + '\n')) + assert list(read_records('-')) == [RECORDS[0]] + + +@pytest.mark.parametrize( + 'line, fmt', + [ + ('{"url": "http://example.com/"}', 'jsonl'), + ('com,example)/ 20240101000000 {"url": "http://example.com/"}', 'cdxj'), + ('url\tfilename\toffset\tlength', 'tsv'), + ('url,filename,offset,length', 'csv'), + ], +) +def test_sniff_format(line, fmt): + assert sniff_format(line) == fmt + + +def test_parquet_round_trip(tmp_path): + pytest.importorskip('pyarrow') + path = str(tmp_path / 'records.parquet') + assert format_from_path(path) == 'parquet' + assert write_records(iter(RECORDS), path, 'parquet') == 2 + assert list(read_records(path)) == RECORDS + + # selected fields, missing values are left out when reading + write_records(iter(RECORDS), path, 'parquet', fields=['url', 'status', 'missing']) + assert list(read_records(path)) == [ + {'url': 'http://example.com/', 'status': '200'}, {'url': 'http://example.com/x', 'status': '404'}, + ] + + +def test_parquet_many_batches(tmp_path, monkeypatch): + pytest.importorskip('pyarrow') + import pyarrow.parquet + + monkeypatch.setattr('cdx_toolkit.record_io.PARQUET_BATCH_SIZE', 3) + records = [{'url': f'http://example.com/{i}', 'offset': i} for i in range(7)] + path = str(tmp_path / 'records.parquet') + + assert write_records(iter(records), path, 'parquet') == 7 + assert pyarrow.parquet.ParquetFile(path).metadata.num_row_groups == 3 + assert [r['offset'] for r in read_records(path)] == [str(i) for i in range(7)] # all columns are strings + + +def test_parquet_needs_a_file(tmp_path): + with pytest.raises(ValueError, match='stdout'): + write_records(iter(RECORDS), '-', 'parquet') + + +def test_cdx_is_an_alias_of_cdxj(tmp_path): + path = str(tmp_path / 'records.cdx') + assert format_from_path(path) == 'cdxj' + write_records(iter(RECORDS), path, 'cdx') + assert open(path).read().startswith('com,example)/ 20240101000000 {') + assert list(read_records(path, 'cdx')) == RECORDS + + +@pytest.mark.parametrize( + 'path, glob', + [ + ('https://example.org/index.cdx.gz?signature=x', False), + ('https://example.org/indexes/*.gz', True), + ('s3://bucket/cdx-0000?.gz', True), + ('-', False), + ], +) +def test_is_glob(path, glob): + assert is_glob(path) == glob + + +def test_no_globs_over_http(): + with pytest.raises(ValueError, match='globs do not work with http'): + list(read_records('https://data.commoncrawl.org/cc-index/collections/CC-MAIN-2024-30/indexes/cdx-*.gz')) + + +@pytest.mark.parametrize('fmt', ['csv', 'tsv', 'parquet']) +def test_fields_that_appear_later_are_kept(tmp_path, fmt): + if fmt == 'parquet': + pytest.importorskip('pyarrow') + records = [{'url': 'http://example.com/'}, {'url': 'http://example.com/x', 'redirect': 'http://example.com/y'}] + path = str(tmp_path / f'out.{fmt}') + assert write_records(iter(records), path, fmt) == 2 + assert list(read_records(path)) == [dict(records[0], **({} if fmt == 'parquet' else {'redirect': ''})), records[1]] + + +@pytest.mark.parametrize('fmt', ['csv', 'tsv']) +def test_fixed_schema_with_null_values(tmp_path, fmt): + """SQL results: every record has all columns, NULL is None (empty in CSV/TSV, left out in JSONL).""" + records = [{'url': 'http://example.com/', 'redirect': None}, {'url': 'http://example.com/x', 'redirect': 'y'}] + path = str(tmp_path / f'out.{fmt}') + write_records(iter(records), path, fmt, fixed_schema=True) + assert list(read_records(path)) == [{'url': 'http://example.com/', 'redirect': ''}, records[1]] + + path = str(tmp_path / 'out.jsonl') + write_records(iter(records), path, 'jsonl', fixed_schema=True) + assert list(read_records(path)) == [{'url': 'http://example.com/'}, records[1]] + + +def test_csv_writer_does_not_drop_fields(): + writer = RecordWriter(io.StringIO(), 'csv') + writer.write({'url': 'a'}) + with pytest.raises(ValueError): + writer.write({'url': 'b', 'redirect': 'c'}) + + +def test_csv_of_capture_objects(tmp_path): + """CDX API captures are CaptureObjects (mappings, not dicts).""" + from cdx_toolkit import CaptureObject + + captures = [CaptureObject({'url': 'http://example.com/', 'status': '200'}, wb=None), + CaptureObject({'url': 'http://example.com/x', 'status': '200', 'redirect': 'y'}, wb=None)] + path = str(tmp_path / 'out.csv') + write_records(iter(captures), path, 'csv') + assert list(read_records(path)) == [ + {'url': 'http://example.com/', 'status': '200', 'redirect': ''}, + {'url': 'http://example.com/x', 'status': '200', 'redirect': 'y'}, + ] + + +def test_overlaps_local(tmp_path): + (tmp_path / 'in').mkdir() + (tmp_path / 'in' / 'a.cdx').write_text('x\n') + a = str(tmp_path / 'in' / 'a.cdx') + assert overlaps(a, a) + assert overlaps(str(tmp_path / 'in' / '.' / 'a.cdx'), a) + assert overlaps(a, str(tmp_path / 'in' / '*.cdx')) + assert overlaps(str(tmp_path / 'in'), str(tmp_path / 'in' / '*.cdx')) # a directory with the inputs + assert overlaps(str(tmp_path / 'in' / 'b.cdx'), str(tmp_path / 'in')) # a file in the input directory + assert not overlaps(str(tmp_path / 'in' / 'b.cdx'), str(tmp_path / 'in' / '*.cdx')) # no such input (yet) + assert not overlaps(str(tmp_path / 'in2'), str(tmp_path / 'in')) + assert not overlaps('-', a) and not overlaps(a, '-') + assert not overlaps('memory://in/a.cdx', a) + + os.link(a, tmp_path / 'hard.cdx') + assert overlaps(str(tmp_path / 'hard.cdx'), a) + try: + os.symlink(a, tmp_path / 'link.cdx') + except OSError: # Windows without the privilege + return + assert overlaps(str(tmp_path / 'link.cdx'), str(tmp_path / 'in' / '*.cdx')) + + +def test_overlaps_remote(): + assert overlaps('memory://in/a.cdx', 'memory://in/a.cdx') + assert overlaps('memory://in/a.cdx', 'memory://in/*.cdx') + assert overlaps('memory://in/sub/a.cdx', 'memory://in/') + assert overlaps('memory://in', 'memory://in/*.cdx') + assert not overlaps('memory://in/a.jsonl', 'memory://in/*.cdx') + assert not overlaps('memory://in2/a.cdx', 'memory://in/a.cdx') + + +def test_is_empty_dir(tmp_path): + assert is_empty_dir(str(tmp_path / 'missing')) + assert is_empty_dir(str(tmp_path)) + (tmp_path / 'f').write_text('x') + assert not is_empty_dir(str(tmp_path)) + assert not is_empty_dir(str(tmp_path / 'f')) diff --git a/tests/unit/test_selectors.py b/tests/unit/test_selectors.py new file mode 100644 index 0000000..1f3a386 --- /dev/null +++ b/tests/unit/test_selectors.py @@ -0,0 +1,124 @@ +"""The URL selectors select the same captures from every backend (CDX files, the columnar index with DuckDB).""" +import json +from urllib.parse import urlsplit + +import pytest + +from cdx_toolkit.cli import main +from tests.conftest import requires_duckdb + + +# url, SURT key (as in Common Crawl's indexes, written out instead of computed by the code under test), +# registered domain +CAPTURES = [ + ('https://example.com/', 'com,example)/', 'example.com'), + ('https://www.example.com/w', 'com,example)/w', 'example.com'), + ('https://blog.example.com/', 'com,example,blog)/', 'example.com'), + ('https://a.blog.example.com/', 'com,example,blog,a)/', 'example.com'), + ('https://ablog.example.com/', 'com,example,ablog)/', 'example.com'), + ('https://blogexample.com/', 'com,blogexample)/', 'blogexample.com'), + ('https://example.com.au/', 'au,com,example)/', 'example.com.au'), + # path and query boundaries of URL prefixes (the SURT key drops a trailing slash) + ('https://example.org/blog', 'org,example)/blog', 'example.org'), + ('https://example.org/blog/', 'org,example)/blog', 'example.org'), + ('https://example.org/blog/entry', 'org,example)/blog/entry', 'example.org'), + ('https://example.org/blogger', 'org,example)/blogger', 'example.org'), + ('https://example.org/blog?x=1', 'org,example)/blog?x=1', 'example.org'), + # ports, and hosts that only look similar + ('https://example.net:8443/', 'net,example:8443)/', 'example.net'), + ('https://www.example.net/', 'net,example)/', 'example.net'), + ('https://sub.example.net:8443/p', 'net,example,sub:8443)/p', 'example.net'), + ('https://example.net.evil.test/', 'test,evil,net,example)/', 'evil.test'), + ('https://notexample.net/', 'net,notexample)/', 'notexample.net'), +] + + +@pytest.fixture(scope='module') +def indexes(tmp_path_factory): + """The same captures as a CDX file and as a columnar (parquet) index.""" + import duckdb + + base = tmp_path_factory.mktemp('indexes') + lines = sorted( + f'{key} 20240801120000 ' + f'{json.dumps({"url": url, "filename": "a.warc.gz", "offset": str(i), "length": "1"})}\n' + for i, (url, key, _) in enumerate(CAPTURES) + ) + (base / 'index.cdx').write_text(''.join(lines)) + + path = base / 'columnar' / 'crawl=CC-MAIN-2024-33' / 'subset=warc' + path.mkdir(parents=True) + values = ', '.join( + f"('{key}', '{url}', '{urlsplit(url).hostname}', '{domain}', '{domain.rsplit('.', 1)[-1]}', {i})" + for i, (url, key, domain) in enumerate(CAPTURES) + ) + duckdb.connect().execute(f''' + COPY (SELECT *, array_to_string(list_reverse(string_split(url_host_name, '.')), '.') AS url_host_name_reversed, + TIMESTAMP '2024-08-01 12:00:00' AS fetch_time, 'a.warc.gz' AS warc_filename, + 1 AS warc_record_length, NULL::VARCHAR AS content_mime_type, + NULL::VARCHAR AS content_mime_detected, NULL::SMALLINT AS fetch_status, + NULL::VARCHAR AS content_digest, NULL::VARCHAR AS content_languages, + NULL::VARCHAR AS content_charset, NULL::VARCHAR AS fetch_redirect + FROM (VALUES {values}) AS t(url_surtkey, url, url_host_name, url_host_registered_domain, url_host_tld, + warc_record_offset)) + TO '{path}/part-0.parquet' (FORMAT parquet) + ''') + return str(base / 'index.cdx'), str(base / 'columnar' / '*' / '*' / '*.parquet') + + +@requires_duckdb +@pytest.mark.parametrize('selector, expected', [ + (['--domain', 'example.com'], [ + 'https://a.blog.example.com/', 'https://ablog.example.com/', 'https://blog.example.com/', + 'https://example.com/', 'https://www.example.com/w', + ]), + (['--domain', 'blog.example.com'], ['https://a.blog.example.com/', 'https://blog.example.com/']), + (['--domain', 'a.blog.example.com'], ['https://a.blog.example.com/']), + (['--domain', 'example.com.au'], ['https://example.com.au/']), + (['--domain', 'Example.COM.'], [ + 'https://a.blog.example.com/', 'https://ablog.example.com/', 'https://blog.example.com/', + 'https://example.com/', 'https://www.example.com/w', + ]), + (['--hostname', 'blog.example.com'], ['https://blog.example.com/']), + # exact hosts, also with www. (which SURT keys drop) + (['--hostname', 'example.com'], ['https://example.com/']), + (['--hostname', 'www.example.com'], ['https://www.example.com/w']), + (['--hostname', 'WWW.Example.com'], ['https://www.example.com/w']), + (['--hostname', 'www.example.com', '--url', 'blog.example.com/*'], + ['https://blog.example.com/', 'https://www.example.com/w']), + (['--hostname', 'www.example.com', '--domain', 'example.com.au'], + ['https://example.com.au/', 'https://www.example.com/w']), + (['--url', '*.blog.example.com'], ['https://a.blog.example.com/', 'https://blog.example.com/']), + (['--url', 'blog.example.com/*'], ['https://blog.example.com/']), + # URL prefixes keep a trailing / or ? as a boundary (like the CDX API) + (['--url', 'example.org/blog/*'], ['https://example.org/blog/entry']), + (['--url', 'example.org/blog*'], [ + 'https://example.org/blog', 'https://example.org/blog/', 'https://example.org/blog/entry', + 'https://example.org/blog?x=1', 'https://example.org/blogger', + ]), + (['--url', 'example.org/blog?*'], ['https://example.org/blog?x=1']), + (['--url', 'example.org/blog/?*'], ['https://example.org/blog?x=1']), + # hosts with a port + (['--hostname', 'example.net'], ['https://example.net:8443/']), + (['--hostname', 'sub.example.net'], ['https://sub.example.net:8443/p']), + (['--domain', 'example.net'], [ + 'https://example.net:8443/', 'https://sub.example.net:8443/p', 'https://www.example.net/', + ]), + (['--url', '*.example.net'], [ + 'https://example.net:8443/', 'https://sub.example.net:8443/p', 'https://www.example.net/', + ]), + (['--url', 'example.net:8443/*'], ['https://example.net:8443/']), +]) +def test_selectors_match_the_same_captures_on_every_backend(indexes, capsys, selector, expected): + for index in indexes: + main(['index', index, '--yes', '--output-format', 'csv', '--fields', 'url'] + selector) + assert sorted(capsys.readouterr().out.splitlines()[1:]) == expected, index + + +@requires_duckdb +def test_count_is_capped_by_the_limit_on_every_backend(indexes, capsys): + for index in indexes: + for limit, expected in ((None, '5'), ('2', '2'), ('9', '5')): + limit_args = ['--limit', limit] if limit else [] + main(['index', index, '--yes', '--domain', 'example.com', '--count'] + limit_args) + assert capsys.readouterr().out.strip() == expected, (index, limit) diff --git a/tests/unit/test_sql_builder.py b/tests/unit/test_sql_builder.py new file mode 100644 index 0000000..811456a --- /dev/null +++ b/tests/unit/test_sql_builder.py @@ -0,0 +1,110 @@ +"""Building the guided SQL queries for the columnar index (no engine).""" +import pytest + +from cdx_toolkit.indexes import IndexQuery +from cdx_toolkit.indexes.backends.sql_builder import build_sql, escape_sql_literal + + +def test_build_query_hostnames(): + q = build_sql(IndexQuery(hostnames=['example.com', 'test.org']), 'ccindex') + assert "subset = 'warc'" in q + assert "url_host_tld = 'com'" in q + assert "url_host_tld = 'org'" in q + assert "url_host_name = 'example.com'" in q + assert "url_host_name = 'test.org'" in q + assert 'warc_filename, warc_record_offset, warc_record_length' in q + assert 'crawl IN' not in q + assert 'LIMIT' not in q + + +def test_build_query_domains_only(): + q = build_sql(IndexQuery(domains=['example.com']), 'ccindex') + assert ("(url_host_name_reversed >= 'com.example' AND url_host_name_reversed < 'com.example/' " + "AND (url_host_name_reversed = 'com.example' OR url_host_name_reversed LIKE 'com.example.%'))") in q + assert "url_host_tld = 'com'" in q + assert 'url_host_name =' not in q + assert 'url_surtkey' not in q.split('WHERE', 1)[1] + + +def test_build_query_hostnames_and_domains(): + q = build_sql(IndexQuery(hostnames=['www.example.com'], domains=['example.org']), 'ccindex') + assert "(url_host_name_reversed = 'com.example.www' AND url_host_name = 'www.example.com')" in q + assert "url_host_name_reversed LIKE 'org.example.%'" in q + assert "url_host_tld = 'com'" in q + assert "url_host_tld = 'org'" in q + + +def test_build_query_requires_host_or_domain(): + with pytest.raises(ValueError): + build_sql(IndexQuery(), 'ccindex') + + +def test_build_query_with_crawls(): + q = build_sql(IndexQuery(hostnames=['example.com']), 'ccindex', crawls=['CC-MAIN-2025-33', 'CC-MAIN-2025-30']) + assert "crawl IN ('CC-MAIN-2025-33', 'CC-MAIN-2025-30')" in q + + +def test_build_query_limit(): + assert 'LIMIT 10' in build_sql(IndexQuery(hostnames=['example.com'], limit=10), 'ccindex') + assert 'LIMIT' not in build_sql(IndexQuery(hostnames=['example.com']), 'ccindex') + + +def test_build_query_orders_by_default(): + q = build_sql(IndexQuery(hostnames=['example.com']), 'ccindex') + assert 'ORDER BY warc_filename, warc_record_offset' in q + + +def test_build_query_order_by_false(): + assert 'ORDER BY' not in build_sql(IndexQuery(hostnames=['example.com']), 'ccindex', order_by=False) + + +def test_build_query_order_by_precedes_limit(): + q = build_sql(IndexQuery(hostnames=['example.com'], limit=10), 'ccindex') + assert q.index('ORDER BY') < q.index('LIMIT') + + +def test_escape_sql_literal_valid(): + assert escape_sql_literal('example.com') == "'example.com'" + assert escape_sql_literal('CC-MAIN-2025-33') == "'CC-MAIN-2025-33'" + + +@pytest.mark.parametrize('bad', ["a';DROP TABLE x;--", "a' OR '1'='1", 'a b', "a'", "', '"]) +def test_escape_sql_literal_rejects_injection(bad): + with pytest.raises(ValueError): + escape_sql_literal(bad) + + +def test_build_query_rejects_injection_hostname(): + with pytest.raises(ValueError): + build_sql(IndexQuery(hostnames=["example.com'; DROP TABLE ccindex; --"]), 'ccindex') + + +def test_resolve_crawl_names(): + from unittest.mock import patch + + from cdx_toolkit.indexes.backends.sql_builder import resolve_crawl_names + + endpoints = [f'https://index.commoncrawl.org/CC-MAIN-2024-{week}-index' for week in (30, 33, 38)] + with patch('cdx_toolkit.indexes.backends.sql_builder.get_cc_endpoints', return_value=endpoints): + assert resolve_crawl_names('2') == ['CC-MAIN-2024-33', 'CC-MAIN-2024-38'] + assert resolve_crawl_names('CC-MAIN-2024-30') == ['CC-MAIN-2024-30'] + + +def test_build_query_domain_www_is_the_domain(): + # like the SURT keys (and --domain of the CDX files): www. is not part of the domain + q = build_sql(IndexQuery(domains=['www.example.com']), 'ccindex') + assert "url_host_name_reversed = 'com.example' OR url_host_name_reversed LIKE 'com.example.%'" in q + + +def test_build_query_ip_addresses_without_reversed_host(): + q = build_sql(IndexQuery(hostnames=['1.2.3.4'], domains=['10.0.0.1'], urls=['1.2.3.4/x*']), 'ccindex') + assert 'url_host_name_reversed' not in q + assert "url_host_name = '1.2.3.4'" in q + assert "url_host_registered_domain IN ('10.0.0.1', '0.0.1', '0.1')" in q + assert "url_surtkey LIKE '4,3,2,1)/x%'" in q + + +def test_build_query_url_with_host_range(): + q = build_sql(IndexQuery(urls=['www.example.com/blog/*']), 'ccindex') + assert ("(url_host_name_reversed >= 'com.example' AND url_host_name_reversed < 'com.example/' " + "AND url_surtkey LIKE 'com,example)/blog/%' ESCAPE '\\')") in q diff --git a/tests/unit/test_sql_indexes.py b/tests/unit/test_sql_indexes.py new file mode 100644 index 0000000..ceb09cb --- /dev/null +++ b/tests/unit/test_sql_indexes.py @@ -0,0 +1,422 @@ +import datetime +import json +from unittest.mock import patch + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index +from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend +from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend +from cdx_toolkit.indexes.backends.athena import AthenaBackend, AthenaLocation +from cdx_toolkit.indexes.backends.duckdb import DuckDbBackend +from cdx_toolkit.indexes.backends.sql import normalize_row +from cdx_toolkit.indexes.backends.sql_builder import check_language, query_conditions +from cdx_toolkit.record_io import read_records + +from tests.conftest import TEST_DATA_PATH, requires_duckdb + + +ATHENA = 'athena://athena.us-east-1.amazonaws.com:443/ccindex?S3OutputLocation=s3://my-bucket/athena/' + + +@pytest.mark.parametrize( + 'location, backend_name, backend, backend_location', + [ + ('cc', 'auto', CdxApiBackend, None), + ('cc-url-index-s3', 'auto', DuckDbBackend, 's3://commoncrawl/cc-index/table/cc-main/warc/'), + ('cc-url-index-hf', 'auto', DuckDbBackend, 'hf://buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/'), + ('./index/part-0.parquet', 'auto', DuckDbBackend, './index/part-0.parquet'), + ('s3://bucket/index/*.parquet', 'auto', DuckDbBackend, 's3://bucket/index/*.parquet'), + ('s3://bucket/index/', 'duckdb', DuckDbBackend, 's3://bucket/index/'), + (ATHENA, 'auto', AthenaBackend, ATHENA), + ('s3://bucket/indexes/cdx-00000.gz', 'auto', CdxFilesBackend, 's3://bucket/indexes/cdx-00000.gz'), + ('./indexes/*.cdx', 'cdx-files', CdxFilesBackend, './indexes/*.cdx'), + ('https://example.com/cdx-00000', 'auto', CdxApiBackend, None), + ('https://example.com/cdx-00000', 'cdx-files', CdxFilesBackend, 'https://example.com/cdx-00000'), + ('https://example.com/cdx.php', 'cdx-api', CdxApiBackend, None), + ], +) +def test_open_index_backend(location, backend_name, backend, backend_location): + resolved = open_index(location, backend_name) + assert isinstance(resolved, backend) + if backend_location: + assert resolved.location == backend_location + + +@pytest.mark.parametrize( + 'location, backend_name, error', + [ + ('cc', 'duckdb', 'cannot be queried with DuckDB'), + ('s3://bucket/index/', 'athena', 'invalid Athena location'), + ('athena://athena.us-east-1.amazonaws.com/ccindex', 'auto', 'S3OutputLocation'), + ('./indexes/*.cdx', 'cdx-api', 'is not a CDX API'), + ('cc', 'cdx-files', 'is not a path of CDX files'), + (ATHENA, 'cdx-files', 'is not a path of CDX files'), + ], +) +def test_open_index_backend_errors(location, backend_name, error): + with pytest.raises(UsageError, match=error): + open_index(location, backend_name) + + +def test_athena_location(): + assert AthenaLocation.parse(ATHENA) == AthenaLocation('us-east-1', 'ccindex', 'ccindex', 's3://my-bucket/athena/') + assert AthenaLocation.parse( + 'athena://athena.eu-west-1.amazonaws.com/mydb?S3OutputLocation=s3://b/p/&table=urls' + ) == AthenaLocation('eu-west-1', 'mydb', 'urls', 's3://b/p/') + + +def test_normalize_row(): + row = { + 'url': 'https://example.com/', + 'fetch_time': datetime.datetime(2024, 7, 13, 10, 22, 33), + 'fetch_status': 200, + 'warc_record_offset': 10, + 'content_languages': None, + 'my_column': 'x', + } + assert normalize_row(row) == { + 'url': 'https://example.com/', 'timestamp': '20240713102233', 'status': '200', 'offset': '10', + 'languages': None, 'my_column': 'x', + } + # Athena returns strings + assert normalize_row({'fetch_time': '2024-07-13 10:22:33.000'}) == {'timestamp': '20240713102233'} + + +@pytest.mark.parametrize( + 'query, error', + [ + (IndexQuery(urls=['ex*ample.com']), 'invalid URL pattern'), + (IndexQuery(sql='SELECT 1', urls=['example.com/*']), '--url cannot be combined with --sql'), + (IndexQuery(domains=['example.com'], filters=['=status:200']), '--filter cannot be used'), + (IndexQuery(url_file='urls.csv'), 'use a custom --sql query that joins the index with a table'), + (IndexQuery(domains=['example.com'], languages=['german']), 'invalid language'), + (IndexQuery(domains=['example.com'], languages=['en']), 'use ISO 639-3 codes'), + (IndexQuery(domains=['example.com'], to='2024-01'), 'invalid timestamp'), + (IndexQuery(sql='SELECT 1', from_ts='2024'), '--from cannot be combined with --sql'), + (IndexQuery(), 'needs --url, --hostname, --domain or --sql'), + ( + IndexQuery(sql='SELECT 1', domains=['example.com'], limit=1), + '--domain, --limit cannot be combined with --sql', + ), + ], +) +def test_sql_validate(query, error): + with pytest.raises(UsageError, match=error): + DuckDbBackend('s3://bucket/index/').validate(query) + + +def test_duckdb_guided_query_prunes_crawls(): + backend = DuckDbBackend('s3://bucket/index/') + query = IndexQuery(hostnames=['www.example.com'], domains=['example.org'], crawl='CC-MAIN-2024-33', limit=5) + + with patch('cdx_toolkit.indexes.backends.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): + plan = backend.plan(query) + sql = plan.sql + + assert plan.n_crawls == 1 + assert "read_parquet(['s3://bucket/index/crawl=CC-MAIN-2024-33/subset=warc/*.parquet']" in sql + assert 'crawl IN' not in sql # pruned by the glob + assert ("((url_host_name_reversed = 'com.example.www' AND url_host_name = 'www.example.com') " + "OR (url_host_name_reversed >= 'org.example' AND url_host_name_reversed < 'org.example/' " + "AND (url_host_name_reversed = 'org.example' OR url_host_name_reversed LIKE 'org.example.%')))") in sql + assert 'url_surtkey, fetch_time, url,' in sql + assert sql.rstrip().endswith('LIMIT 5') + + +def test_guided_query_url_patterns(): + query = IndexQuery(urls=['example.com/a_b*', 'https://www.example.org/', '*.example.net'], domains=['example.com']) + sql = AthenaBackend(ATHENA).plan(query).sql + + assert "(url_host_tld = 'com' OR url_host_tld = 'net' OR url_host_tld = 'org')" in sql + range_com = "url_host_name_reversed >= 'com.example' AND url_host_name_reversed < 'com.example/'" + range_org = "url_host_name_reversed >= 'org.example' AND url_host_name_reversed < 'org.example/'" + range_net = "url_host_name_reversed >= 'net.example' AND url_host_name_reversed < 'net.example/'" + assert (f"(({range_com} AND (url_host_name_reversed = 'com.example' " + "OR url_host_name_reversed LIKE 'com.example.%')) " + f"OR ({range_org} AND url_surtkey = 'org,example)/') " + f"OR ({range_com} AND url_surtkey LIKE 'com,example)/a\\_b%' ESCAPE '\\') " + f"OR ({range_net} AND url_surtkey LIKE 'net,example)%' ESCAPE '\\') " + f"OR ({range_net} AND url_surtkey LIKE 'net,example,%' ESCAPE '\\') " + f"OR ({range_net} AND url_surtkey LIKE 'net,example:%' ESCAPE '\\'))") in sql + + +def test_guided_query_time_range_and_languages(): + query = IndexQuery(hostnames=['example.com'], from_ts='2024', to='20240410', languages=['deu', 'fra']) + sql = AthenaBackend(ATHENA).plan(query).sql + + assert "fetch_time >= TIMESTAMP '2024-01-01 00:00:00'" in sql + assert "fetch_time < TIMESTAMP '2024-04-11 00:00:00'" in sql + assert ("(content_languages = 'deu' OR content_languages LIKE 'deu,%' " + "OR content_languages = 'fra' OR content_languages LIKE 'fra,%')") in sql + + +def test_athena_guided_query_and_cost_guard(): + backend = AthenaBackend(ATHENA) + query = IndexQuery(domains=['example.com']) + + plan = backend.plan(query) + sql = plan.sql + assert plan.n_crawls is None + assert 'FROM ccindex' in sql + + # not restricted to crawls -> confirmation needed, refused in non-interactive mode + with patch('cdx_toolkit.indexes.backends.sql.sys.stdin') as stdin, patch.object(backend, 'iter_sql_rows') as rows: + stdin.isatty.return_value = False + with pytest.raises(SystemExit, match='--yes'): + list(backend.records(query)) + rows.assert_not_called() + + backend.assume_yes = True + with patch.object(backend, 'iter_sql_rows', return_value=iter([{'url': 'https://example.com/'}])) as rows: + assert list(backend.records(query)) == [{'url': 'https://example.com/'}] + assert rows.call_args.args == (sql,) + + +def test_sql_count_and_explain(): + backend = AthenaBackend(ATHENA, assume_yes=True) + query = IndexQuery(sql='SELECT url FROM ccindex WHERE crawl = \'CC-MAIN-2024-33\';\n') + + with patch.object(backend, 'iter_sql_rows', return_value=iter([{'n': '42'}])) as rows: + assert backend.count(query) == 42 + assert rows.call_args.args == ( + "SELECT count(*) AS n FROM (SELECT url FROM ccindex WHERE crawl = 'CC-MAIN-2024-33') AS q", + ) + + assert 'SELECT url FROM ccindex' in backend.explain(query) + + +@pytest.fixture +def parquet_index(tmp_path): + """A tiny columnar index, partitioned like Common Crawl's (crawl=.../subset=warc/*.parquet).""" + import duckdb + + base = tmp_path / 'index' + rows = [ + ('CC-MAIN-2024-33', 'org,example)/', 'https://example.org/', 'example.org', 'www.example.org', 'a.warc.gz', 30), + ('CC-MAIN-2024-33', 'com,example)/', 'https://example.com/', 'example.com', 'example.com', 'b.warc.gz', 20), + ('CC-MAIN-2024-33', 'com,example)/x', 'https://example.com/x', 'example.com', 'example.com', 'a.warc.gz', 10), + ('CC-MAIN-2024-30', 'com,example)/', 'https://example.com/', 'example.com', 'example.com', 'c.warc.gz', 0), + ] + con = duckdb.connect() + for crawl in ('CC-MAIN-2024-30', 'CC-MAIN-2024-33'): + path = base / f'crawl={crawl}' / 'subset=warc' + path.mkdir(parents=True) + values = ', '.join( + f"('{r[1]}', TIMESTAMP '2024-08-01 12:00:00', '{r[2]}', 'text/html', 200, '{r[3]}', '{r[4]}', " + f"'{r[3].split('.')[-1]}', '{r[5]}', {r[6]}, 100)" + for r in rows if r[0] == crawl + ) + con.execute(f''' + COPY (SELECT *, + array_to_string(list_reverse(string_split(url_host_name, '.')), '.') AS url_host_name_reversed, + NULL::VARCHAR AS content_mime_detected, NULL::VARCHAR AS content_digest, + NULL::VARCHAR AS content_languages, NULL::VARCHAR AS content_charset, + NULL::VARCHAR AS fetch_redirect + FROM (VALUES {values}) AS t(url_surtkey, fetch_time, url, content_mime_type, + fetch_status, url_host_registered_domain, url_host_name, url_host_tld, warc_filename, + warc_record_offset, warc_record_length)) + TO '{path}/part-0.parquet' (FORMAT parquet) + ''') + return base + + +@requires_duckdb +def test_index_duckdb_end_to_end(parquet_index, tmp_path, capsys): + output = tmp_path / 'out.jsonl' + with patch('cdx_toolkit.indexes.backends.sql.resolve_crawl_names', return_value=['CC-MAIN-2024-33']): + main([ + 'index', str(parquet_index), '--index-backend', 'duckdb', '--domain', 'example.com', + '--crawl', 'CC-MAIN-2024-33', '-o', str(output), + ]) + + records = [json.loads(line) for line in output.read_text().splitlines()] + # sorted by WARC file and offset + assert [(r['filename'], r['offset']) for r in records] == [('a.warc.gz', '10'), ('b.warc.gz', '20')] + assert records[0] == { + 'urlkey': 'com,example)/x', 'timestamp': '20240801120000', 'url': 'https://example.com/x', 'mime': 'text/html', + 'status': '200', 'filename': 'a.warc.gz', 'offset': '10', 'length': '100', 'crawl': 'CC-MAIN-2024-33', + } + + +@requires_duckdb +def test_index_duckdb_parquet_glob_sql_and_count(parquet_index, tmp_path, capsys, monkeypatch): + files = f'{parquet_index}/*/*/*.parquet' + + # all crawls -> needs --yes in non-interactive mode + main(['index', files, '--hostname', 'www.example.org', '--yes', '--output-format', 'csv', '--fields', 'url,crawl']) + assert capsys.readouterr().out.splitlines() == ['url,crawl', 'https://example.org/,CC-MAIN-2024-33'] + + main(['index', files, '--domain', 'example.com', '--count', '--yes']) + assert capsys.readouterr().out == '3\n' + + sql = tmp_path / 'query.sql' + sql.write_text(f"SELECT url, warc_filename FROM read_parquet('{files}') WHERE warc_record_offset > 15;") + main(['index', files, '--sql', str(sql), '--yes', '--output-format', 'csv']) + assert sorted(capsys.readouterr().out.splitlines()) == [ + 'https://example.com/,b.warc.gz', 'https://example.org/,a.warc.gz', 'url,filename' + ] + + main(['index', files, '--sql', str(sql), '--dry-run']) + assert 'WHERE warc_record_offset > 15' in capsys.readouterr().out + + +@requires_duckdb +def test_index_duckdb_time_range_and_language(tmp_path, capsys): + import duckdb + + path = tmp_path / 'crawl=CC-MAIN-2024-33' / 'subset=warc' + path.mkdir(parents=True) + rows = [ + # url, fetch_time (UTC), content_languages + ('https://example.com/a', '2024-08-01 23:30:00', 'deu'), + ('https://example.com/b', '2024-08-02 00:30:00', 'eng,deu'), + ('https://example.com/c', '2024-08-03 12:00:00', 'deu,eng'), + ('https://example.com/d', '2024-08-04 12:00:00', None), + ] + values = ', '.join( + f"('{url}', 'com,example)/{url[-1]}', TIMESTAMPTZ '{t}+00', {repr(lang) if lang else 'NULL'}, 'example.com', " + f"'example.com', 'com', 'a.warc.gz', {i}, 100)" + for i, (url, t, lang) in enumerate(rows) + ) + duckdb.connect().execute(f''' + COPY (SELECT *, array_to_string(list_reverse(string_split(url_host_name, '.')), '.') AS url_host_name_reversed, + NULL::VARCHAR AS content_mime_type, + NULL::VARCHAR AS content_mime_detected, NULL::SMALLINT AS fetch_status, + NULL::VARCHAR AS content_digest, NULL::VARCHAR AS content_charset, NULL::VARCHAR AS fetch_redirect + FROM (VALUES {values}) AS t(url, url_surtkey, fetch_time, content_languages, url_host_registered_domain, + url_host_name, url_host_tld, warc_filename, warc_record_offset, warc_record_length)) + TO '{path}/part-0.parquet' (FORMAT parquet) + ''') + files = f'{tmp_path}/*/*/*.parquet' + + def urls(*args): + main(['index', files, '--domain', 'example.com', '--yes', '--output-format', 'csv', '--fields', 'url,timestamp'] + + list(args)) + return capsys.readouterr().out.splitlines()[1:] + + # timestamps are UTC, the day of --to is included + assert urls('--from', '20240802', '--to', '20240803') == [ + 'https://example.com/b,20240802003000', 'https://example.com/c,20240803120000', + ] + # the primary (first) content language + assert urls('--language', 'deu') == ['https://example.com/a,20240801233000', 'https://example.com/c,20240803120000'] + assert urls('--language', 'eng', '--language', 'deu', '--to', '2024080123') == [ + 'https://example.com/a,20240801233000', + ] + + +@pytest.mark.parametrize('code, language', [('eng', 'eng'), ('DEU', 'deu'), (' fra ', 'fra'), ('nds', 'nds')]) +def test_check_language(code, language): + assert check_language(code) == language + + +def test_languages_are_normalized_in_the_query(): + sql = AthenaBackend(ATHENA).plan(IndexQuery(hostnames=['example.com'], languages=['DEU'])).sql + assert "content_languages = 'deu'" in sql + + +@requires_duckdb +def test_index_duckdb_url_patterns(parquet_index, capsys): + files = f'{parquet_index}/*/*/*.parquet' + + def urls(*args): + main(['index', files, '--yes', '--output-format', 'csv', '--fields', 'url'] + list(args)) + return sorted(capsys.readouterr().out.splitlines()[1:]) + + assert urls('--url', 'example.com/x') == ['https://example.com/x'] + assert urls('--url', 'example.com/*') == ['https://example.com/', 'https://example.com/', 'https://example.com/x'] + assert urls('--url', '*.example.org', '--url', 'example.com/x') == ['https://example.com/x', 'https://example.org/'] + + +@requires_duckdb +def test_sql_without_url_column_to_repackage(parquet_index, tmp_path, capsys): + """A --sql query that selects only the WARC location is enough for repackaging.""" + from tests.fake_warc_server import FakeWarcServer + + server = FakeWarcServer('https://data.example.org') + capture, = server.add_captures('a.warc.gz', ['https://example.com/x']) + files = f'{parquet_index}/*/*/*.parquet' + sql = tmp_path / 'query.sql' + sql.write_text( + f"SELECT warc_filename, {capture['offset']} AS warc_record_offset, {capture['length']} AS warc_record_length " + f"FROM read_parquet('{files}') WHERE url = 'https://example.com/x' AND crawl = 'CC-MAIN-2024-33'" + ) + captures = tmp_path / 'captures.jsonl' + main(['index', files, '--sql', str(sql), '--yes', '-o', str(captures)]) + assert [json.loads(line) for line in captures.read_text().splitlines()] == [ + {'filename': 'a.warc.gz', 'offset': capture['offset'], 'length': capture['length']} + ] + with patch('cdx_toolkit.repackage.readers.http.myrequests_get', side_effect=server.get): + main(['repackage', str(captures), '--prefix', str(tmp_path / 'out'), + '--warc-download-prefix', 'https://data.example.org']) + assert len(list(tmp_path.glob('out-*.warc.gz'))) == 1 + + with pytest.raises(SystemExit) as exc: + main(['index', files, '--sql', str(sql), '--yes', '--url-fgrep', 'example']) + assert exc.value.code == 2 + assert '--url-fgrep and --url-fgrepv need captures with a url' in capsys.readouterr().err + + +@requires_duckdb +@pytest.mark.parametrize('to, included', [ + ('20240801', ['23:59:59.500']), + ('20240801235959', ['23:59:59.500']), + ('2024080123595', ['23:59:59.500']), + ('20240802', ['23:59:59.500', 'next day']), +]) +def test_end_timestamp_includes_fractional_seconds(to, included): + import duckdb + + rows = {'23:59:59.500': '2024-08-01 23:59:59.500', 'next day': '2024-08-02 00:00:00'} + con = duckdb.connect() + con.execute('CREATE TABLE t (name VARCHAR, fetch_time TIMESTAMP)') + con.executemany('INSERT INTO t VALUES (?, ?)', list(rows.items())) + condition, = query_conditions(IndexQuery(to=to)) + assert sorted(r[0] for r in con.execute(f'SELECT name FROM t WHERE {condition}').fetchall()) == included + + +def test_end_timestamp_overflow(): + with pytest.raises(UsageError, match='invalid timestamp'): + query_conditions(IndexQuery(to='9999')) + + +@requires_duckdb +def test_example_homepages_sql(tmp_path): + """examples/homepages.sql (the --sql example of the README), on a local copy of the index's columns.""" + import duckdb + + sql = (TEST_DATA_PATH.parent.parent / 'examples' / 'homepages.sql').read_text() + s3_files = 's3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-33/subset=warc/*.parquet' + assert s3_files in sql + rows = [ + # url, url_host_tld, url_path, url_query, fetch_status + ('https://www.vatican.va/', 'va', '/', None, 200), # a homepage + ('https://www.vatican.va/content/', 'va', '/content/', None, 200), + ('https://www.vatican.va/?lang=it', 'va', '/', 'lang=it', 200), + ('http://vatican.va/', 'va', '/', None, 301), + ('https://example.com/', 'com', '/', None, 200), + ] + values = ', '.join( + f"('{url}', '{tld}', '{path}', {repr(query) if query else 'NULL'}, {status}, 'text/html', " + f"TIMESTAMP '2024-08-01 12:00:00', 'crawl-data/a.warc.gz', {i}, 100)" + for i, (url, tld, path, query, status) in enumerate(rows) + ) + index = tmp_path / 'part-0.parquet' + duckdb.connect().execute(f''' + COPY (SELECT * FROM (VALUES {values}) AS t(url, url_host_tld, url_path, url_query, fetch_status, + content_mime_type, fetch_time, warc_filename, warc_record_offset, warc_record_length)) + TO '{index}' (FORMAT parquet) + ''') + query = tmp_path / 'homepages.sql' + query.write_text(sql.replace(s3_files, str(index))) + out = tmp_path / 'homepages.csv' + + main(['index', str(index), '--sql', str(query), '--yes', '-o', str(out)]) + + captures = list(read_records(str(out), 'csv')) + assert [c['url'] for c in captures] == ['https://www.vatican.va/'] + # the fields cdxt repackage needs + assert {k: captures[0][k] for k in ('filename', 'offset', 'length')} == { + 'filename': 'crawl-data/a.warc.gz', 'offset': '0', 'length': '100'} diff --git a/tests/unit/test_timeutils.py b/tests/unit/test_timeutils.py index 7f090d8..86f6f6c 100644 --- a/tests/unit/test_timeutils.py +++ b/tests/unit/test_timeutils.py @@ -7,6 +7,9 @@ def test_padding(): assert timeutils.pad_timestamp('1998') == '19980101000000' assert timeutils.pad_timestamp_up('199802') == '19980228235959' assert timeutils.pad_timestamp_up('199812') == '19981231235959' + assert timeutils.pad_timestamp_up('202402') == '20240229235959' # leap year + assert timeutils.pad_timestamp_up('20260410') == '20260410235959' # the day is kept + assert timeutils.pad_timestamp_up('2026041012') == '20260410125959' def test_time_timestamp(): diff --git a/tests/unit/test_usage_errors.py b/tests/unit/test_usage_errors.py new file mode 100644 index 0000000..d29d575 --- /dev/null +++ b/tests/unit/test_usage_errors.py @@ -0,0 +1,140 @@ +"""Usage errors and small edge cases of `cdxt index` / `cdxt repackage` and their helpers.""" +import io +import runpy +import sys +from unittest.mock import patch + +import pytest + +from cdx_toolkit.cli import main +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index +from cdx_toolkit.indexes.backends.cdx_api import CdxApiBackend +from cdx_toolkit.indexes.backends.cdx_files import CdxFilesBackend, load_url_patterns +from cdx_toolkit.indexes.patterns import domain_entry_to_surts, url_pattern_to_surts +from cdx_toolkit.indexes.backends.duckdb import DuckDbBackend +from cdx_toolkit.indexes.backends.sql_builder import quote_sql_string, sql_timestamp +from cdx_toolkit.record_io import RecordWriter, import_pyarrow, read_records, write_records + +from tests.conftest import TEST_DATA_PATH + +CDX_PATH = str(TEST_DATA_PATH / 'repackage' / 'filtered_CC-MAIN-2024-30_cdx-00187.gz') + + +def test_python_m_cdx_toolkit_cli(monkeypatch, capsys): + monkeypatch.setattr(sys, 'argv', ['cdxt', '--version']) + with pytest.raises(SystemExit): + runpy.run_module('cdx_toolkit.cli', run_name='__main__') + assert capsys.readouterr().out.strip() + + +@pytest.mark.parametrize( + 'args, error', + [ + (['index', 'cc', '--url', 'example.com/*', '-o', 'out.txt'], 'cannot detect the output format of out.txt'), + (['index', CDX_PATH, '--domain', 'mnhn.fr', '--to', '2024-01'], 'invalid timestamp'), + (['index', CDX_PATH, '--url', 'ex*ample.com/*'], '* can only be at the start or the end'), + (['index', 'cc-url-index-s3', '--domain', 'example.com', '--to', '20241332'], "invalid timestamp '20241332'"), + (["index", "/data/it's.parquet", '--domain', 'example.com'], 'invalid index location'), + (['index', 'ia', '--url', 'example.com/*', '--filter', '=status:200'], + "--filter '=status:200': ia does not support the filter ="), + ], +) +def test_usage_errors(capsys, args, error): + with pytest.raises(SystemExit) as e: + main(args) + assert e.value.code == 2 + assert error in capsys.readouterr().err + + +@pytest.mark.parametrize( + 'command', [['index', 'cc', '--url', 'x.org/*', '-o', 'x.parquet'], ['repackage', 'x.parquet']] +) +def test_parquet_without_pyarrow(capsys, command): + with patch('cdx_toolkit.cli.index.import_pyarrow', side_effect=RuntimeError('needs pyarrow')), \ + patch('cdx_toolkit.record_io.import_pyarrow', side_effect=RuntimeError('needs pyarrow')): + with pytest.raises(SystemExit): + main(command) + assert 'needs pyarrow' in capsys.readouterr().err + + +def test_import_pyarrow_missing(monkeypatch): + monkeypatch.setitem(sys.modules, 'pyarrow', None) + with pytest.raises(RuntimeError, match=r'cdx_toolkit\[parquet\]'): + import_pyarrow() + + +def test_cdx_api_dry_run_and_cc_sort(capsys): + main(['index', 'cc', '--domain', 'example.com', '--crawl', '3', '--limit', '5', '--dry-run']) + assert capsys.readouterr().out == 'cc: url=example.com limit=5 matchType=domain\ncrawls: 3\n' + + with patch('cdx_toolkit.CDXFetcher') as fetcher: + CdxApiBackend('cc', cc_sort='ascending')._fetcher(IndexQuery(urls=['x'])) + assert fetcher.call_args.kwargs == {'source': 'cc', 'cc_sort': 'ascending'} + + +def test_cdx_files_edge_cases(tmp_path, capsys): + with pytest.raises(UsageError, match='unknown --url-file-type'): + CdxFilesBackend(CDX_PATH).validate(IndexQuery(url_file='u.txt', url_file_type='regex')) + with pytest.raises(FileNotFoundError, match='no files found'): + CdxFilesBackend(str(tmp_path / 'cdx-*.gz')).files() + + # rows of a URL file without the column are skipped (after the first one) + path = tmp_path / 'urls.jsonl' + path.write_text('{"url": "example.com/a"}\n{"other": 1}\n{"url": "example.com/b"}\n') + assert load_url_patterns(str(path)) == ['example.com/a', 'example.com/b'] + + # the time range excludes all lines; it is shown in the dry run + assert CdxFilesBackend(CDX_PATH).count(IndexQuery(domains=['mnhn.fr'], to='2023')) == 0 + main(['index', CDX_PATH, '--domain', 'mnhn.fr', '--from', '2024', '--dry-run']) + assert 'timestamps: 20240101000000 to -' in capsys.readouterr().out + + +def test_pattern_errors(): + with pytest.raises(UsageError): + url_pattern_to_surts('ex*ample.com/*') + with pytest.raises(UsageError, match='no wildcards'): + domain_entry_to_surts('*.example.com') + + +def test_sql_helpers(): + with pytest.raises(UsageError): + sql_timestamp('20241332') + with pytest.raises(ValueError, match='control characters'): + quote_sql_string('a\nb') + with pytest.raises(UsageError): + DuckDbBackend("x'.parquet") + + +def test_record_io_unknown_formats(tmp_path): + with pytest.raises(ValueError, match='unknown format'): + RecordWriter(io.StringIO(), 'xml') + path = tmp_path / 'r.jsonl' + path.write_text('{"url": "x"}\n') + with pytest.raises(ValueError, match='unknown format'): + list(read_records(str(path), 'xml')) + + +def test_parquet_batches_exactly_full(tmp_path, monkeypatch): + pytest.importorskip('pyarrow') + monkeypatch.setattr('cdx_toolkit.record_io.PARQUET_BATCH_SIZE', 2) + path = str(tmp_path / 'r.parquet') + assert write_records(iter([{'url': str(i)} for i in range(4)]), path, 'parquet') == 4 + assert [r['url'] for r in read_records(path)] == ['0', '1', '2', '3'] + + +@pytest.mark.parametrize('index', ['ia', 'https://example.org/cdx', 'index.cdx', 'cc-url-index-s3', 'x.parquet', + 'athena://athena.us-east-1.amazonaws.com/ccindex?S3OutputLocation=s3://b/']) +@pytest.mark.parametrize('option', [['--cc-mirror', 'https://mirror.example.org/'], ['--cc-sort', 'ascending']]) +def test_cc_options_only_for_cc(capsys, index, option): + with pytest.raises(SystemExit) as e: + main(['index', index, '--domain', 'example.com', '--dry-run'] + option) + assert e.value.code == 2 + assert f'{option[0]} can only be used with the Common Crawl CDX API (cc)' in capsys.readouterr().err + + with pytest.raises(UsageError, match=option[0]): + open_index(index, **{option[0][2:].replace('-', '_'): option[1]}) + + +def test_cc_options_are_passed_on(): + backend = open_index('cc', cc_mirror='https://mirror.example.org/', cc_sort='ascending', assume_yes=True) + assert (backend.cc_mirror, backend.cc_sort) == ('https://mirror.example.org/', 'ascending') diff --git a/tests/unit/test_warc.py b/tests/unit/test_warc.py index 8a99fa5..7371eae 100644 --- a/tests/unit/test_warc.py +++ b/tests/unit/test_warc.py @@ -3,6 +3,14 @@ from unittest.mock import Mock import pytest +# the tests of the deprecated writer (see test_writer_is_deprecated) +pytestmark = pytest.mark.filterwarnings('ignore:cdx_toolkit.warc.get_writer:FutureWarning') + + +def test_writer_is_deprecated(tmp_path): + with pytest.warns(FutureWarning, match='use cdx_toolkit.repackage.repackage'): + cdx_toolkit.warc.get_writer(str(tmp_path / 'out'), None, {}) + def test_wb_redir_to_original(): location = 'https://web.archive.org/web/20110209062054id_/http://commoncrawl.org/' @@ -11,14 +19,14 @@ def test_wb_redir_to_original(): @requires_aws_s3 -def test_fetch_warc_record_from_s3(): +def test_fetch_warc_record_from_s3(s3_env): record = cdx_toolkit.warc.fetch_warc_record( - capture={ + capture=s3_env.capture({ 'url': 'https://bibliotheque.missiondefrance.fr/index.php?lvl=bulletin_display&id=319', 'filename': 'crawl-data/CC-MAIN-2024-30/segments/1720763514759.37/warc/CC-MAIN-20240716142214-20240716172214-00337.warc.gz', # noqa: E501 'offset': 111440525, 'length': 9754, - }, + }), warc_download_prefix='s3://commoncrawl', ) record_content = record.content_stream().read().decode(errors='ignore')