CLI refactor: cdxt index + cdxt repackage

.gitattributes

@@ -0,0 +1,2 @@
+# test fixtures are compared byte for byte (e.g. WARC payload digests): no line-ending conversion
+tests/data/** -text

.github/workflows/ci-slow.yaml

@@ -2,6 +2,14 @@ name: CI-slow
 
 on: workflow_dispatch
 
+# live tests (CDXT_LIVE_TESTS=http,s3,athena,hf): the HTTP requests to the CDX APIs and data.commoncrawl.org, S3,
+# Athena and DuckDB against AWS, and the Hugging Face Storage Buckets (HF_TOKEN: writes to
+# hf://buckets/commoncrawl/commoncrawl-ci-temp). The permissions are needed to get AWS credentials via GitHub's OIDC
+# token endpoint
+permissions:
+  id-token: write
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   # 'queue: max' ensures old runs finish in order instead of being canceled
@@ -42,8 +50,17 @@ jobs:
           pip install setuptools
 
       - name: Install cdx_toolkit
-        run: pip install .[test]
+        run: pip install ".[all]"
+
+      - name: Configure AWS credentials from OIDC
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: arn:aws:iam::837454214164:role/GitHubActions-Role
+          aws-region: us-east-1
 
       - name: Run tests
+        env:
+          CDXT_LIVE_TESTS: 'http,s3,athena,hf'
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: |
           make test_coverage

.github/workflows/ci.yaml

@@ -8,9 +8,8 @@ on:
     branches:
     - main
 
-# These permissions are needed to interact with AWS S3 via GitHub's OIDC Token endpoint
+# the tests make no remote requests (they are mocked); the live tests run in ci-slow.yaml
 permissions:
-  id-token: write
   contents: read
   pull-requests: read
 
@@ -39,7 +38,7 @@ jobs:
           - python-version: '3.14'
             os: ubuntu-latest
             env:
-              LOGLEVEL=DEBUG
+              CDXT_LOGLEVEL=DEBUG
             EXTRA: true
           - python-version: '3.14'
             os: macos-latest
@@ -68,21 +67,8 @@ jobs:
           pip install setuptools
 
       - name: Install cdx_toolkit
-        run: pip install .[all]
+        run: pip install ".[all]"
 
-      - name: Configure AWS credentials from OIDC (disabled for forks)
-        if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
-        uses: aws-actions/configure-aws-credentials@v6
-        with:
-          role-to-assume: arn:aws:iam::837454214164:role/GitHubActions-Role
-          aws-region: us-east-1
-
-      - name: Disable S3 unit tests for Python 3.8 (botocore requires Python 3.9+)
-        if: ${{ startsWith(matrix.python-version, '3.8') }}
-        uses: actions/github-script@v9
-        with:
-          script: |
-            core.exportVariable('CDXT_DISABLE_S3_TESTS', '1')
       - name: Set environment variables for faster unit tests (requests are mocked)
         uses: actions/github-script@v9
         with:
@@ -93,8 +79,7 @@ jobs:
             core.exportVariable('CDXT_CC_INDEX_MIN_RETRY_INTERVAL', '0.01')
             core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01')
             core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01')
-            core.exportVariable('DISABLE_ATHENA_TESTS', '1')
-            core.exportVariable('LOGLEVEL', 'DEBUG')
+            core.exportVariable('CDXT_LOGLEVEL', 'DEBUG')
 
       - name: Lint code
         run: |
@@ -140,7 +125,7 @@ jobs:
           pip install setuptools
 
       - name: Install cdx_toolkit (minimal)
-        run: pip install .[test]
+        run: pip install ".[test]"
 
       - name: Set environment variables for faster unit tests (requests are mocked)
         uses: actions/github-script@v9
@@ -152,8 +137,7 @@ jobs:
             core.exportVariable('CDXT_CC_INDEX_MIN_RETRY_INTERVAL', '0.01')
             core.exportVariable('CDXT_CC_DATA_MIN_RETRY_INTERVAL', '0.01')
             core.exportVariable('CDXT_IA_MIN_RETRY_INTERVAL', '0.01')
-            core.exportVariable('DISABLE_ATHENA_TESTS', '1')
-            core.exportVariable('LOGLEVEL', 'DEBUG')
+            core.exportVariable('CDXT_LOGLEVEL', 'DEBUG')
 
       - name: test minimal
         run: |

.travis.yml

@@ -23,7 +23,7 @@ matrix:
     arch: s390x  # unreliable in travis
   include:
   - python: 3.5
-    env: LOGLEVEL=DEBUG  # same time as normal
+    env: CDXT_LOGLEVEL=DEBUG  # same time as normal
   - python: 3.8
     arch: arm64  # 50% extra runtime
   - python: 3.8

CHANGELOG.md

@@ -1,3 +1,64 @@
+- unreleased
+	+ new command line interface: `cdxt index` and `cdxt repackage` (see README)
+	+ the command lines of 0.9.39 (`cdxt --cc iter URL`, `size`, `warc`) are deprecated (to be removed in a future version): they run
+	  with the new code and print a suggested new command line; `iter` output, `--get` and `size --details` are kept;
+	  `warc` writes `TEST-001.warc.gz` (was `TEST-000000.extracted.warc.gz`) with the records copied byte-for-byte
+	  (no `WARC-Source-URI`/`WARC-Source-Range` headers), no file without captures, and exits with status 1 if
+	  captures could not be fetched; `iter --all-fields --csv` is a usage error (was a traceback)
+	+ `cdxt index`: `--hostname`, `--domain`, repeatable `--url`, `-o/--output` (local or remote),
+	  JSONL/CDXJ/CSV/TSV output, all fields by default, `--count` replaces `size`
+	+ `cdxt repackage`: reads captures from a file or stdin, human-readable `--size`, `--is-part-of`, `--description`
+	+ `--cc-sort` is now passed on to the CDX fetcher
+	+ the log level variable is `CDXT_LOGLEVEL` (`LOGLEVEL` still works), like cdx_toolkit's other environment variables
+	+ `cdxt index` queries Common Crawl's columnar URL index with DuckDB (`cc-url-index-s3`, `cc-url-index-hf`, `*.parquet`)
+	  or AWS Athena (`athena://...`): `--hostname`/`--domain`/`--crawl`/`--limit` or your own `--sql`, cost confirmation (`--yes`), `--dry-run`;
+	  `--index-backend cdx-api|cdx-files|duckdb|athena` overrides the backend detected from INDEX
+	+ `cdxt repackage` copies WARC records byte-for-byte with the async WARC fetcher (from the feat/warc-range-sources branch):
+	  reads from https://, s3:// and hf:// (Hugging Face Storage Buckets), writes to local, s3:// or hf://,
+	  `--processes`, `--parallel-readers`, `--add-metadata-record`, glob input; output files are now named `<prefix>-001.warc.gz`
+	  (`--wb` wayback replays too, which now work with `--parallel-readers` and `--add-metadata-record`); with
+	  `--processes`, the processes share one file counter
+	  and write the final file names directly, so each `warcinfo` record names its own file
+	+ `hf://` WARC reads resolve each WARC file once and read its records from the signed CDN URL
+	  (one HF Resolvers rate-limit request per WARC file instead of per record)
+	+ `cdxt index` reads CDX files (local or remote, globs): `--url`/`--hostname`/`--domain`/`--url-file` matched as SURT
+	  prefixes (`--url-file-type domain` for host lists), one output per CDX file for an `-o` path without extension
+	  (a new or empty directory), `--processes`; `cdxt index` refuses an output that would overwrite its input
+	+ SQL indexes: `--url` patterns (matched against `url_surtkey`), `--from`/`--to` (fetch time), `--language` (primary content language, three-letter ISO 639-3 codes)
+	+ SQL indexes: `--hostname`, `--domain` and `--url` select by `url_host_name_reversed`, which follows the sort order
+	  of the index, so DuckDB (and Athena) skip the row groups of other hosts by their min/max statistics; `--domain` no
+	  longer reads `url_surtkey` (one Parquet file: 3 s instead of 87 s over S3)
+	+ `--output-format cdx` / `--input-format cdx` as another name for `cdxj`; clear error for globs over http(s)
+	+ Parquet output for `cdxt index` and Parquet input for `cdxt repackage` (`cdx_toolkit[parquet]`)
+	+ `cdxt repackage`: default `--prefix REPACKAGE` (was `TEST`), no WARC file is written without captures,
+	  a full URL in `warc_filename`/`filename` is used as it is (the `warc_url` column is gone); stops (exit status 2)
+	  if WARC files with the same prefix exist already, instead of overwriting them (also with `--wb`, which
+	  numbered its files after the existing ones before)
+	+ fixed `--to` with a day (e.g. `--to 20240410`), which was padded to the end of the month, and February in leap years
+	+ `cdxt index ia --filter` with a filter the Internet Archive does not support (`=`, `~`, `!=`, `!~`) is a usage error
+	  (exit status 2) before any request, instead of a traceback while the captures are read
+	+ fixed DNS errors of hosts that worked before: they are retried again (like connection errors) instead of failing at once
+	+ removed the deprecated `cdx_iter` and `cdx_size` scripts
+	+ S3 requests use Common Crawl's recommended AWS settings (us-east-1, adaptive retries, 100000 attempts, virtual
+	  addressing, no payload signing, 2 concurrent requests) unless configured by the user; `CDXT_AWS_DEFAULTS=0` turns them off;
+	  also for Athena and, as far as DuckDB has equivalents, for DuckDB (fixes DuckDB's `http_timeout`, which is in seconds)
+	+ `cdxt repackage --wb` checks the captures before replaying them: `url` (http/https), `timestamp` and `status`
+	  are needed, and Common Crawl captures are refused (as `cdxt --cc warc` refused `--wb`)
+	+ Python API of the new commands: `cdx_toolkit.indexes.open_index()` with `IndexQuery` (`records()`, `count()`, `explain()`)
+	  and `cdx_toolkit.repackage.repackage()`; `cdxt index` and `cdxt repackage` run the same code
+	+ deprecated `cdx_toolkit.warc.get_writer()` and `CDXToolkitWARCWriter` (FutureWarning): use `repackage()`
+	+ new extras: `athena`, `duckdb`, `hf` (`huggingface_hub>=1.7.2`, the first version whose `HfFileSystem` handles
+	  Storage Buckets), `parquet`; `fsspec` and `aiofiles` are now required
+
+- 0.9.39
+	+ `CDXToolkitWARCWriter` (`cdx_toolkit.warc.get_writer()`) supports the context manager protocol (`with ... as writer:`),
+	  so the WARC file is closed, and an S3 upload committed, also on errors; `close()` still works (#70)
+	+ WARC files can be written to S3 (and other fsspec file systems); `cdx_toolkit[s3]` extra (#67)
+	+ settings are read from `CDXT_*` environment variables (`settings.py`), common CLI code moved to `utils.py`,
+	  the CLI and the example close the WARC writer (#67)
+	+ weekly GitHub Action that checks Common Crawl's CDX index server (#60)
+	+ PyPI releases with a manually dispatched trusted-publisher workflow (#69)
+
 - 0.9.38
 	+ deprecated support for py3.7 and py.3.8
 	+ added support for py3.13 and py3.14

CONTRIBUTING.md

@@ -34,19 +34,46 @@ make test
 make unit
 ```
 
-By default, all remote requests are mocked. To change this behaviour and actually call remote APIs (if you run this from a whitelisted IP address), the following environment variable can be set:
+By default, all remote requests are mocked: the CDX API and HTTP downloads are served from recorded data in `tests/data`, S3 is a local [moto](https://github.com/getmoto/moto) server (installed with `.[all]`; without it, the S3 tests are skipped), and the Hugging Face reads are faked. The mocked requests still wait for the per-host rate limits ([docs/settings.md](docs/settings.md)); the CI shortens them:
 
 ```bash
-export DISABLE_MOCK_RESPONSES=1
+export CDXT_DEFAULT_MIN_RETRY_INTERVAL=0.01 CDXT_CC_INDEX_MIN_RETRY_INTERVAL=0.01 \
+       CDXT_CC_DATA_MIN_RETRY_INTERVAL=0.01 CDXT_IA_MIN_RETRY_INTERVAL=0.01
 ```
 
+To test services live instead, set `CDXT_LIVE_TESTS` to a comma-separated list of them, or `all` (unset the short intervals above, the live servers rate-limit):
+
+| Service | Live tests |
+|---|---|
+| `http` | the CDX API (index.commoncrawl.org, web.archive.org) and HTTP downloads (data.commoncrawl.org); needs an allowed IP address |
+| `s3` | the S3 tests against AWS, and DuckDB queries of Common Crawl's index on S3; needs AWS credentials |
+| `athena` | the Athena queries; needs AWS credentials with Athena access |
+| `hf` | `cdxt index cc-url-index-hf`, and `cdxt repackage` reading from `hf://buckets/commoncrawl/commoncrawl` and writing to `hf://buckets/commoncrawl/commoncrawl-ci-temp` (`CDXT_TEST_HF_BUCKET`); writing needs a Hugging Face token (`HF_TOKEN` or `hf auth login`) |
+
+```bash
+CDXT_LIVE_TESTS=s3,athena,hf make test
+```
+
+The tests that can run live are marked `live` and `live_<service>`, so `-m` selects only them:
+
+```bash
+CDXT_LIVE_TESTS=all pytest -m live tests      # all live tests
+CDXT_LIVE_TESTS=s3 pytest -m live_s3 tests    # only the S3 tests
+```
+
+The marker only selects the tests; without the service in `CDXT_LIVE_TESTS`, they are skipped (`s3`, `athena`, `hf`) or run with mocked responses (`http`, and `s3` against moto).
+
+The CI workflow of pull requests runs the mocked tests only; the `CI-slow` workflow (started manually) runs all live tests (`http,s3,athena,hf`) with AWS credentials and `HF_TOKEN`.
+
+The WARC records that the `cdxt repackage` tests download (from `data.commoncrawl.org` or `s3://commoncrawl`) are recorded in `tests/data/repackage/warc_records.jsonl`, and DuckDB queries a small local index built from the same captures (the `local_url_index` fixture in `tests/conftest.py`).
+
 If the remote APIs change, new mock data can be semi-automatically collected by setting another environment variable,  running corresponding unit tests, and overwriting existing mock data in `tests/data/mock_responses`:
 
 ```bash
-# set environment variable (DISABLE_MOCK_RESPONSES should not be set)
-export SAVE_MOCK_RESPONSES=./tmp/mock_responses
+# set environment variable (CDXT_LIVE_TESTS should not have http)
+export CDXT_SAVE_MOCK_RESPONSES=./tmp/mock_responses
 
-# run the test for what mock data should be saved to $SAVE_MOCK_RESPONSES/<test_file>/<test_func>.jsonl
+# run the test for what mock data should be saved to $CDXT_SAVE_MOCK_RESPONSES/<test_file>/<test_func>.jsonl
 pytest tests/test_cli.py::test_basics
 ```
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: init test clean_coverage test_coverage lint distclean distcheck dist install
+.PHONY: init test example clean_coverage test_coverage lint distclean distcheck dist install
 
 init36:
 	# packages are deprecating support, so this uses exact versions
@@ -12,6 +12,9 @@ unit:
 
 test:
 	PYTHONPATH=. py.test --doctest-modules cdx_toolkit tests -v -v
+
+# runs against the live Common Crawl index (not part of `test`: connections from CI runners time out)
+example:
 	PYTHONPATH=. examples/iter-and-warc.py
 
 clean_coverage:
@@ -20,7 +23,7 @@ clean_coverage:
 test_coverage: clean_coverage
 	#	PYTHONPATH=. coverage run -a --source=cdx_toolkit,examples examples/iter-and-warc.py
 	# -rA to see all output, pass or fail
-	# LOGLEVEL=DEBUG
+	# CDXT_LOGLEVEL=DEBUG
 	# -vvvv -s
 	PYTHONPATH=. py.test -rA -s --doctest-modules --cov-report=xml --cov-append --cov cdx_toolkit tests -v -v
 	coverage report