From 88bc80bc881da24bf84a75f020c831b5b9e21969 Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:05:06 +0200 Subject: [PATCH 01/15] Add evidence report analysis mode --- src/wayparam/config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/wayparam/config.py b/src/wayparam/config.py index a0c7554..cbc4c7a 100644 --- a/src/wayparam/config.py +++ b/src/wayparam/config.py @@ -21,7 +21,14 @@ from .wayback import CdxOptions AnalysisMode = Literal[ - "history", "params", "summary", "timeline", "changes", "topology", "cooccurrence" + "history", + "params", + "summary", + "timeline", + "changes", + "topology", + "cooccurrence", + "report", ] TimelineGranularity = Literal["year", "month"] From ceafca99321f7a0c332805d0e409f71dc71bc97e Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:05:10 +0200 Subject: [PATCH 02/15] Add versioned evidence report records --- src/wayparam/analysis.py | 73 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/wayparam/analysis.py b/src/wayparam/analysis.py index 3483e0b..01eb039 100644 --- a/src/wayparam/analysis.py +++ b/src/wayparam/analysis.py @@ -15,6 +15,7 @@ import httpx +from . import __version__ from .config import AnalysisMode, RunConfig, TimelineGranularity from .core import Budget, DomainStats, ProgressCallback, client_kwargs from .filters import is_boring @@ -26,6 +27,7 @@ log = logging.getLogger("wayparam") _PROGRESS_EVERY = 1000 +REPORT_SCHEMA = "wayparam-evidence-report/v1" def _min_ts(current: str | None, candidate: str | None) -> str | None: @@ -528,8 +530,19 @@ def records_for( *, timeline_granularity: TimelineGranularity = "year", compare_periods: tuple[str, str] | None = None, + report_complete: bool | None = None, + report_bounded: bool = False, ) -> list[dict]: """Return deterministic serializable records for one analysis view.""" + if mode == "report": + return report_records( + history, + timeline_granularity=timeline_granularity, + compare_periods=compare_periods, + complete=report_complete, + bounded=report_bounded, + ) + if mode == "history": return [ { @@ -590,6 +603,62 @@ def records_for( ] +def report_records( + history: DomainHistory, + *, + timeline_granularity: TimelineGranularity = "year", + compare_periods: tuple[str, str] | None = None, + complete: bool | None = None, + bounded: bool = False, +) -> list[dict]: + """Build a versioned JSONL evidence bundle from one historical aggregation.""" + sections: list[AnalysisMode] = [ + "summary", + "history", + "params", + "timeline", + "topology", + "cooccurrence", + ] + if compare_periods is not None: + sections.append("changes") + + manifest = { + "type": "report_manifest", + "schema": REPORT_SCHEMA, + "generator": {"name": "wayparam", "version": __version__}, + "domain": history.domain, + "source": "wayback", + "evidence_scope": "archive-index", + "sections": list(sections), + "timeline_granularity": timeline_granularity, + "compare_periods": list(compare_periods) if compare_periods else None, + "complete": complete, + "bounded_capture_budget": bounded, + "limitations": [ + "Archive-index evidence only; no live-target state is inferred.", + "Bounded or incomplete runs may omit archived evidence.", + ], + } + + out = [manifest] + for section in sections: + for record in records_for( + history, + section, + timeline_granularity=timeline_granularity, + compare_periods=compare_periods, + ): + out.append( + { + "type": "report_record", + "section": section, + "record": record, + } + ) + return out + + def _compact_counter(value: dict[str, int]) -> str: return ",".join(f"{key}:{count}" for key, count in value.items()) or "-" @@ -719,15 +788,19 @@ def write_analysis_files(result: HistoryRunResult, cfg: RunConfig, mode: Analysi if not cfg.write_files: return cfg.outdir.mkdir(parents=True, exist_ok=True) + stats_by_domain = {item.domain: item for item in result.stats} for domain in cfg.domains: history = result.analyses.get(domain) if history is None: continue + stats = stats_by_domain.get(domain) with open_outfile(analysis_path(cfg.outdir, domain, mode, cfg.out_format)) as fh: for record in records_for( history, mode, timeline_granularity=cfg.timeline_granularity, compare_periods=cfg.compare_periods, + report_complete=stats.complete if stats else None, + report_bounded=cfg.max_results > 0, ): fh.write(format_record(record, cfg.out_format) + "\n") From 2c3f43604b67ea2ede4eb7f083fccf5cd265c2cf Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:05:13 +0200 Subject: [PATCH 03/15] Expose versioned evidence reports in the CLI --- src/wayparam/cli.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/wayparam/cli.py b/src/wayparam/cli.py index fa744b3..702ea24 100644 --- a/src/wayparam/cli.py +++ b/src/wayparam/cli.py @@ -184,6 +184,13 @@ def build_arg_parser() -> argparse.ArgumentParser: const="cooccurrence", help="Aggregate parameter pairs observed together on archived endpoint variants.", ) + analysis.add_argument( + "--report", + dest="analysis", + action="store_const", + const="report", + help="Emit a versioned JSONL evidence bundle from one historical archive pass.", + ) analysis.add_argument( "--changes", nargs=2, @@ -200,7 +207,13 @@ def build_arg_parser() -> argparse.ArgumentParser: "--timeline-granularity", choices=["year", "month"], default="year", - help="Time bucket for --timeline: year or month (default: year).", + help="Time bucket for --timeline/--report: year or month (default: year).", + ) + p.add_argument( + "--report-changes", + nargs=2, + metavar=("BASELINE", "COMPARISON"), + help="Include YYYY or YYYYMM period-change evidence inside --report.", ) # Shared archive query options @@ -415,11 +428,14 @@ def build_config(args: argparse.Namespace) -> RunConfig: else: domains = read_domains(args.list) + raw_compare_periods = args.changes or args.report_changes compare_periods = ( - validate_compare_periods((args.changes[0], args.changes[1])) if args.changes else None + validate_compare_periods((raw_compare_periods[0], raw_compare_periods[1])) + if raw_compare_periods + else None ) analysis_mode: AnalysisMode | None - if compare_periods: + if args.changes: analysis_mode = "changes" else: analysis_mode = args.analysis @@ -501,6 +517,12 @@ def main(argv: list[str] | None = None) -> int: parser.error("--no-files requires --stdout") if args.provenance and args.format != "jsonl": parser.error("--provenance requires --format jsonl") + if args.analysis == "report" and args.format != "jsonl": + parser.error("--report requires --format jsonl") + if args.report_changes and args.analysis != "report": + parser.error("--report-changes requires --report") + if args.report_changes and args.changes: + parser.error("--report-changes cannot be combined with --changes") if args.provenance and (args.analysis or args.changes): parser.error("--provenance cannot be combined with historical analysis modes") if args.provenance and args.source_summary: @@ -524,7 +546,7 @@ def main(argv: list[str] | None = None) -> int: if cfg.analysis and cfg.sources != ("wayback",): parser.error( - "--history/--params/--summary/--timeline/--changes/--topology/--cooccurrence currently require --source wayback" + "--history/--params/--summary/--timeline/--changes/--topology/--cooccurrence/--report currently require --source wayback" ) if cfg.source_summary and len(cfg.sources) < 2: parser.error("--source-summary requires at least two archive sources") @@ -567,15 +589,19 @@ def on_record(rec: UrlRecord) -> None: write_analysis_files(history_result, cfg, cfg.analysis) if args.stdout: + stats_by_domain = {item.domain: item for item in history_result.stats} for domain in cfg.domains: history = history_result.analyses.get(domain) if history is None: continue + stats = stats_by_domain.get(domain) for record in records_for( history, cfg.analysis, timeline_granularity=cfg.timeline_granularity, compare_periods=cfg.compare_periods, + report_complete=stats.complete if stats else None, + report_bounded=cfg.max_results > 0, ): print(format_record(record, cfg.out_format), flush=True) return _report(history_result, cfg, args.stats) From 0b55f09ac3daaa1a7d17006f99124370ee6365d2 Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:06:03 +0200 Subject: [PATCH 04/15] Test versioned evidence report bundles --- tests/test_analysis.py | 97 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 3c2e527..82571fa 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -336,6 +336,79 @@ def test_changes_support_month_periods_and_deterministic_text(): ] + +def test_report_bundle_starts_with_versioned_manifest_and_wraps_sections(): + records = records_for( + _history(), + "report", + timeline_granularity="month", + report_complete=True, + report_bounded=True, + ) + + manifest = records[0] + assert manifest == { + "type": "report_manifest", + "schema": "wayparam-evidence-report/v1", + "generator": {"name": "wayparam", "version": analysis.__version__}, + "domain": "example.com", + "source": "wayback", + "evidence_scope": "archive-index", + "sections": [ + "summary", + "history", + "params", + "timeline", + "topology", + "cooccurrence", + ], + "timeline_granularity": "month", + "compare_periods": None, + "complete": True, + "bounded_capture_budget": True, + "limitations": [ + "Archive-index evidence only; no live-target state is inferred.", + "Bounded or incomplete runs may omit archived evidence.", + ], + } + + wrapped = records[1:] + assert wrapped + assert all(record["type"] == "report_record" for record in wrapped) + assert {record["section"] for record in wrapped} == { + "summary", + "history", + "params", + "timeline", + "topology", + "cooccurrence", + } + summary = next(record for record in wrapped if record["section"] == "summary") + assert summary["record"]["type"] == "summary" + assert summary["record"]["unique_urls"] == 2 + + +def test_report_bundle_can_include_period_changes(): + records = records_for( + _change_history(), + "report", + compare_periods=("2020", "2021"), + ) + + manifest = records[0] + assert manifest["compare_periods"] == ["2020", "2021"] + assert manifest["sections"][-1] == "changes" + changes = [record for record in records if record.get("section") == "changes"] + assert changes[0]["record"]["type"] == "change_summary" + assert any(record["record"].get("status") == "added" for record in changes[1:]) + + +def test_report_bundle_is_compact_jsonl(): + record = records_for(_history(), "report")[0] + encoded = format_record(record, "jsonl") + assert json.loads(encoded) == record + assert " " not in encoded + def test_jsonl_output_is_compact_and_machine_readable(): record = records_for(_history(), "summary")[0] encoded = format_record(record, "jsonl") @@ -408,6 +481,30 @@ def test_history_max_results_caps_accepted_captures(tmp_path): assert stats.complete is False +def test_report_file_uses_report_jsonl_name(tmp_path): + history = _history() + cfg = RunConfig( + domains=["example.com"], + outdir=tmp_path, + write_files=True, + out_format="jsonl", + analysis="report", + max_results=10, + ) + result = HistoryRunResult( + stats=[analysis.DomainStats("example.com", fetched=3, kept=3, complete=False)], + analyses={"example.com": history}, + ) + + write_analysis_files(result, cfg, "report") + + path = tmp_path / "example.com.report.jsonl" + lines = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + assert lines[0]["type"] == "report_manifest" + assert lines[0]["complete"] is False + assert lines[0]["bounded_capture_budget"] is True + + def test_analysis_files_use_mode_specific_names(tmp_path): history = _history() cfg = RunConfig( From fdc4c7aa3a6468251dbe6d96818c590aa39bcc22 Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:06:06 +0200 Subject: [PATCH 05/15] Test evidence report CLI contract --- tests/test_broken_pipe.py | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_broken_pipe.py b/tests/test_broken_pipe.py index 9511d5a..87f87a0 100644 --- a/tests/test_broken_pipe.py +++ b/tests/test_broken_pipe.py @@ -167,6 +167,7 @@ async def capped(_cfg, **_kw): ("--timeline", "timeline"), ("--topology", "topology"), ("--cooccurrence", "cooccurrence"), + ("--report", "report"), ], ) def test_analysis_flags_reach_the_config(flag, mode): @@ -175,6 +176,26 @@ def test_analysis_flags_reach_the_config(flag, mode): assert cfg.analysis == mode + +def test_report_changes_reach_the_config(): + parser = cli.build_arg_parser() + cfg = cli.build_config( + parser.parse_args( + [ + "-d", + "example.com", + "--report", + "--report-changes", + "2020", + "2024", + "--format", + "jsonl", + ] + ) + ) + assert cfg.analysis == "report" + assert cfg.compare_periods == ("2020", "2024") + def test_timeline_granularity_reaches_the_config(): parser = cli.build_arg_parser() cfg = cli.build_config( @@ -206,6 +227,30 @@ def test_invalid_change_periods_are_usage_errors(periods, capsys): assert "change" in capsys.readouterr().err.lower() + +def test_report_requires_jsonl(capsys): + with pytest.raises(SystemExit) as exc: + cli.main(["-d", "example.com", "--report"]) + assert exc.value.code == 2 + assert "--report requires --format jsonl" in capsys.readouterr().err + + +def test_report_changes_require_report(capsys): + with pytest.raises(SystemExit) as exc: + cli.main( + [ + "-d", + "example.com", + "--report-changes", + "2020", + "2024", + "--format", + "jsonl", + ] + ) + assert exc.value.code == 2 + assert "--report-changes requires --report" in capsys.readouterr().err + def test_analysis_views_are_mutually_exclusive(): parser = cli.build_arg_parser() with pytest.raises(SystemExit) as exc: From 262b6e41713a253174902336f539be3dabd49eb2 Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:06:09 +0200 Subject: [PATCH 06/15] Document evidence report work for v1.0.0 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1993d0..abe9cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased +- Added JSONL-only `--report` evidence bundles with the versioned `wayparam-evidence-report/v1` schema, combining summary, endpoint history, parameter history, timeline, topology and co-occurrence from one Wayback metadata pass +- Added optional `--report-changes BASELINE COMPARISON` to include temporal change evidence in the same report bundle + ## 0.9.0 - Added `--topology` historical surface topology analysis, grouping archive evidence by host/path with observed schemes, normalized URL variants, parameter unions and deterministic parameter-set variants - Added `--cooccurrence` to report unordered parameter pairs observed together, with normalized URL, route, capture and first/last-seen evidence From fae5a9695009f5d8ee97d680615e4db905b2bfc1 Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:06:12 +0200 Subject: [PATCH 07/15] Document versioned evidence report bundles --- docs/historical-analysis.md | 44 ++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/historical-analysis.md b/docs/historical-analysis.md index 3d76175..6ed5945 100644 --- a/docs/historical-analysis.md +++ b/docs/historical-analysis.md @@ -3,7 +3,7 @@ wayparam can aggregate capture-level metadata from the Wayback CDX API instead of returning only normalized URLs. -The six analysis views are mutually exclusive: +The individual analysis views are mutually exclusive: ```bash wayparam -d example.com --history @@ -13,6 +13,7 @@ wayparam -d example.com --timeline wayparam -d example.com --changes 2020 2024 wayparam -d example.com --topology wayparam -d example.com --cooccurrence +wayparam -d example.com --report --format jsonl ``` They work with the normal filtering, date-range, subdomain, proxy, rate-limit @@ -210,6 +211,46 @@ Pairs are descriptive archive-index evidence only. Their presence or frequency does not imply dependency between parameters, exploitability or current live behavior. + +## `--report` + +The evidence report is the versioned, machine-readable bundle intended for +long-lived pipelines and research artifacts. It is JSONL-only and reuses one +historical Wayback metadata pass to emit all compatible views together. + +Example: + +```bash +wayparam -d example.com --report --format jsonl --stdout --no-files +``` + +The first line is a `report_manifest` with schema +`wayparam-evidence-report/v1`, the Wayparam generator version, archive source, +domain, evidence scope, included sections, timeline granularity, completeness +state and whether a capture budget was configured. + +Following lines are `report_record` wrappers. Each wrapper identifies its +`section` and contains the unchanged record produced by that individual view. +The default sections are: + +- `summary` +- `history` +- `params` +- `timeline` +- `topology` +- `cooccurrence` + +Optional period-change evidence can be included without a second archive query: + +```bash +wayparam -d example.com --report --report-changes 2020 2024 \ + --format jsonl --stdout --no-files +``` + +Reports are explicitly scoped to archive-index evidence. A bounded or incomplete +run can omit evidence and the manifest records that limitation. + + ## Output files When files are enabled, analysis views do not overwrite the normal URL output. @@ -223,6 +264,7 @@ results/example.com.timeline.txt results/example.com.changes.txt results/example.com.topology.txt results/example.com.cooccurrence.txt +results/example.com.report.jsonl ``` With `--format jsonl`, the extension is `.jsonl`. From 72aa60a7454fee6a49ae727454a83c708c30692b Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:06:17 +0200 Subject: [PATCH 08/15] Document v1.0 evidence report usage --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 043847a..e152738 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ http://www.example.com/?format=FUZZ&retailerId=FUZZ - **Historical intelligence**: first/last seen, capture counts, status/MIME distributions, parameter prevalence - **Temporal surface intelligence**: year/month timelines of archived endpoint and parameter activity - **Surface topology intelligence**: host/path grouping with observed parameter-set variants and parameter co-occurrence +- **Versioned evidence reports**: combine historical views in one JSONL bundle from a single archive metadata pass - Filters “boring” URLs by: - extension blacklist/whitelist - optional path regex exclusion @@ -252,6 +253,19 @@ wayparam -d example.com --cooccurrence --format jsonl --stdout --no-files `--cooccurrence` emits one record for each unordered pair of parameter names observed together on the same normalized endpoint variant, including represented URL count, distinct host/path routes, capture count and first/last seen. It is descriptive archive evidence only and does not assign risk or exploitability. +### 17) Build a versioned evidence report + +```bash +wayparam -d example.com --report --format jsonl --stdout --no-files +wayparam -d example.com --report --report-changes 2020 2024 --format jsonl +``` + +`--report` performs one uncollapsed Wayback metadata pass and emits a +`wayparam-evidence-report/v1` JSONL bundle containing summary, endpoint history, +parameter history, timeline, topology and co-occurrence records. Optional +`--report-changes` adds a two-period comparison to the same bundle. The manifest +records generator version, included sections and completeness/bounded-run context. + Historical modes retrieve capture metadata and automatically disable CDX collapse so that first/last seen, capture counts and temporal activity are meaningful. They can therefore transfer substantially more data than a normal URL run; use `--from`, `--to`, filters or `--max-results` to bound the analysis. --- @@ -348,6 +362,8 @@ See [Archive sources](docs/sources.md) for provider behavior and caveats. * `--changes BASELINE COMPARISON` — compare two YYYY or YYYYMM archive buckets * `--topology` — group archived normalized URL evidence by host/path and parameter-set variants * `--cooccurrence` — aggregate unordered parameter pairs observed on the same archived endpoint variants +* `--report --format jsonl` — emit the versioned evidence bundle from one historical pass +* `--report-changes BASELINE COMPARISON` — optionally include temporal change evidence in a report These views are mutually exclusive. In historical modes, `--max-results` caps accepted capture rows before aggregation. From 7e1b4f7428e025d31007c77a1576749fce71ee4a Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:06:55 +0200 Subject: [PATCH 09/15] Guard evidence reports against text formatting --- src/wayparam/analysis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/wayparam/analysis.py b/src/wayparam/analysis.py index 01eb039..579f411 100644 --- a/src/wayparam/analysis.py +++ b/src/wayparam/analysis.py @@ -668,6 +668,8 @@ def format_record(record: dict, fmt: OutputFormat) -> str: return json.dumps(record, ensure_ascii=False, separators=(",", ":")) kind = record["type"] + if kind in {"report_manifest", "report_record"}: + raise ValueError("evidence reports require jsonl output") if kind == "history": return "\t".join( [ From cd2fd0cd613a6be6e3850880a00c2dcc04565076 Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:06:58 +0200 Subject: [PATCH 10/15] Format evidence report analysis tests --- tests/test_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 82571fa..23af027 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -336,7 +336,6 @@ def test_changes_support_month_periods_and_deterministic_text(): ] - def test_report_bundle_starts_with_versioned_manifest_and_wraps_sections(): records = records_for( _history(), From 57aa33e63ee789e823d18019b699275e31346a39 Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:07:01 +0200 Subject: [PATCH 11/15] Format evidence report CLI tests --- tests/test_broken_pipe.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_broken_pipe.py b/tests/test_broken_pipe.py index 87f87a0..ee11422 100644 --- a/tests/test_broken_pipe.py +++ b/tests/test_broken_pipe.py @@ -176,7 +176,6 @@ def test_analysis_flags_reach_the_config(flag, mode): assert cfg.analysis == mode - def test_report_changes_reach_the_config(): parser = cli.build_arg_parser() cfg = cli.build_config( @@ -227,14 +226,12 @@ def test_invalid_change_periods_are_usage_errors(periods, capsys): assert "change" in capsys.readouterr().err.lower() - def test_report_requires_jsonl(capsys): with pytest.raises(SystemExit) as exc: cli.main(["-d", "example.com", "--report"]) assert exc.value.code == 2 assert "--report requires --format jsonl" in capsys.readouterr().err - def test_report_changes_require_report(capsys): with pytest.raises(SystemExit) as exc: cli.main( From 9e889af0f6d938635c892967f21d7aef9c32b0ab Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:07:33 +0200 Subject: [PATCH 12/15] Document evidence report CLI options --- docs/cli/options.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/cli/options.md b/docs/cli/options.md index 71e5b53..48bf825 100644 --- a/docs/cli/options.md +++ b/docs/cli/options.md @@ -178,6 +178,7 @@ The following flags are mutually exclusive: - `--changes BASELINE COMPARISON`: compare two YYYY or YYYYMM buckets and classify normalized URLs/parameters as added, removed or persisted - `--topology`: group normalized archive evidence by host/path and parameter-set variants - `--cooccurrence`: report unordered parameter pairs observed together with URL/route/capture counts and first/last seen +- `--report`: emit a versioned JSONL evidence bundle combining the compatible historical views from one Wayback metadata pass Historical modes currently require `--source wayback`. They request `timestamp,statuscode,mimetype,original` from Wayback CDX and automatically disable `collapse=urlkey`, because collapsed results cannot provide correct capture counts or first/last-seen dates. @@ -189,8 +190,32 @@ wayparam -d example.com --timeline --timeline-granularity month --format jsonl wayparam -d example.com --changes 2020 2024 --format jsonl wayparam -d example.com --topology --format jsonl wayparam -d example.com --cooccurrence --format jsonl +wayparam -d example.com --report --format jsonl ``` +### `--report` +Emit the stable machine-readable evidence bundle. This mode requires +`--format jsonl` and currently requires `--source wayback`. + +The first JSONL record is a `report_manifest` with schema +`wayparam-evidence-report/v1`, generator version, domain, archive source, +included sections, completeness state and bounded-run context. Following lines +are `report_record` wrappers containing the unchanged records from summary, +history, params, timeline, topology and co-occurrence. + +Use `--report-changes BASELINE COMPARISON` to include the existing period-change +view in the same bundle without a second archive request. + +```bash +wayparam -d example.com --report --format jsonl --stdout --no-files +wayparam -d example.com --report --report-changes 2020 2024 --format jsonl +``` + +### `--report-changes BASELINE COMPARISON` +Include temporal change evidence in `--report`. Both periods must be `YYYY` or +both `YYYYMM`, and the baseline must precede the comparison period. This option +is invalid without `--report`. + ### `--topology` Group accepted historical evidence by host/path. The record includes observed schemes, total captures, normalized URL variants, first/last seen, the union of @@ -216,7 +241,7 @@ records for URL shapes and parameter names. `removed` means absent from the comparison archive bucket, not proven absent from the live target. ### `--timeline-granularity {year|month}` -Choose the bucket size used by `--timeline`. +Choose the bucket size used by `--timeline` and the timeline section of `--report`. **Default:** `year` From f5c31440a5844755e1a6645e906d6bff84e13302 Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:07:37 +0200 Subject: [PATCH 13/15] Document evidence reports in the manpage --- man/wayparam.1 | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/man/wayparam.1 b/man/wayparam.1 index ca1d746..f11289b 100644 --- a/man/wayparam.1 +++ b/man/wayparam.1 @@ -157,6 +157,20 @@ Report unordered query-parameter pairs observed together on normalized archived endpoint variants. Records include URL count, distinct host/path route count, capture count and first/last seen. No risk or exploitability score is inferred. .TP +.BR \-\-report +Emit a versioned JSONL evidence bundle from one uncollapsed Wayback metadata +pass. The bundle begins with a report manifest using schema +.BR wayparam-evidence-report/v1 +and then wraps summary, history, parameter, timeline, topology and +co-occurrence records. Requires +.BR "--format jsonl" . +.TP +.BR \-\-report\-changes " " \fIBASELINE COMPARISON\fR +Include period-change evidence inside +.BR \-\-report . +Both values must use the same YYYY or YYYYMM granularity and BASELINE must +precede COMPARISON. +.TP .BR \-\-source\-summary Compare normalized URL coverage across at least two selected archive sources. The summary reports the union, per-source counts, source-exclusive counts, URLs @@ -372,6 +386,14 @@ Group archived endpoints by host/path structure: wayparam -d example.com --topology --format jsonl .fi .RE +.PP +Build a versioned evidence report: +.PP +.RS +.nf +wayparam -d example.com --report --format jsonl --stdout --no-files +.fi +.RE .SH FILES .TP .I results/.txt @@ -401,6 +423,9 @@ Historical host/path topology with parameter-set variants. .TP .I results/.cooccurrence.{txt,jsonl} Historical parameter-pair co-occurrence evidence. +.TP +.I results/.report.jsonl +Versioned historical evidence report bundle. .SH EXIT STATUS .TP .B 0 From 738ae095bdc320506f6b91a1534eaade1b52b38c Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:09:24 +0200 Subject: [PATCH 14/15] Fix report JSONL compactness test --- tests/test_analysis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 23af027..74354c4 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -406,7 +406,9 @@ def test_report_bundle_is_compact_jsonl(): record = records_for(_history(), "report")[0] encoded = format_record(record, "jsonl") assert json.loads(encoded) == record - assert " " not in encoded + assert ": " not in encoded + assert ", " not in encoded + def test_jsonl_output_is_compact_and_machine_readable(): record = records_for(_history(), "summary")[0] From 6ed624b8277ccc84c93832715301e15c24e26d4d Mon Sep 17 00:00:00 2001 From: Aleff Date: Mon, 21 Sep 2026 17:09:35 +0200 Subject: [PATCH 15/15] Apply Ruff spacing to report CLI tests --- tests/test_broken_pipe.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_broken_pipe.py b/tests/test_broken_pipe.py index ee11422..bbf7331 100644 --- a/tests/test_broken_pipe.py +++ b/tests/test_broken_pipe.py @@ -195,6 +195,7 @@ def test_report_changes_reach_the_config(): assert cfg.analysis == "report" assert cfg.compare_periods == ("2020", "2024") + def test_timeline_granularity_reaches_the_config(): parser = cli.build_arg_parser() cfg = cli.build_config( @@ -232,6 +233,7 @@ def test_report_requires_jsonl(capsys): assert exc.value.code == 2 assert "--report requires --format jsonl" in capsys.readouterr().err + def test_report_changes_require_report(capsys): with pytest.raises(SystemExit) as exc: cli.main( @@ -248,6 +250,7 @@ def test_report_changes_require_report(capsys): assert exc.value.code == 2 assert "--report-changes requires --report" in capsys.readouterr().err + def test_analysis_views_are_mutually_exclusive(): parser = cli.build_arg_parser() with pytest.raises(SystemExit) as exc: