diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index 863608e6..260e5225 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -246,3 +246,19 @@ jobs: ${{ runner.temp }}/host-compatibility-results.xml ${{ runner.temp }}/host-wheel.log retention-days: 30 + + evidence-budget: + # Report-only evidence byte budgets accepted in #273; enforcement is a separate decision. + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - name: Report evidence byte budgets (Refs #273) + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: python3 tools/check_evidence_budget.py --base "$BASE_SHA" --report-only diff --git a/tests/test_evidence_budget.py b/tests/test_evidence_budget.py new file mode 100644 index 00000000..21b120aa --- /dev/null +++ b/tests/test_evidence_budget.py @@ -0,0 +1,651 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from tools import check_evidence_budget as check + +LIMIT_FILE = 512 * 1024 +LIMIT_PR = 2 * 1024 * 1024 + + +def test_file_under_budget_passes(): + files = [("evidence/campaign_x/summary.json", 100 * 1024, 100 * 1024)] + violations, total = check.check_file_budgets( + files, per_file_limit=LIMIT_FILE, per_pr_limit=LIMIT_PR + ) + assert violations == [] + assert total == 100 * 1024 + + +def test_file_exceeding_budget_fails(): + files = [("evidence/campaign_x/large_trace.parquet", 600 * 1024, 600 * 1024)] + violations, total = check.check_file_budgets( + files, per_file_limit=LIMIT_FILE, per_pr_limit=LIMIT_PR + ) + assert len(violations) == 1 + assert "exceeds per-file limit" in violations[0] + assert total == 600 * 1024 + + +def test_aggregate_pr_evidence_budget_fails(): + # 5 files added with 450 KiB each = 2,250 KiB (> 2,048 KiB limit), though each file <= 512 KiB + files = [ + (f"evidence/campaign_x/trace_{i}.json", 450 * 1024, 450 * 1024) + for i in range(5) + ] + violations, total = check.check_file_budgets( + files, per_file_limit=LIMIT_FILE, per_pr_limit=LIMIT_PR + ) + assert len(violations) == 1 + assert "exceeds per-PR net limit" in violations[0] + assert total == 5 * 450 * 1024 + + +def test_modified_file_counts_true_net_addition_not_full_size(): + # A 500 KiB existing file modified with only +10 KiB net growth + files = [("evidence/campaign_x/existing.json", 500 * 1024, 10 * 1024)] + violations, total = check.check_file_budgets( + files, per_file_limit=LIMIT_FILE, per_pr_limit=LIMIT_PR + ) + assert violations == [] + assert total == 10 * 1024 + + +def test_net_addition_with_deletion_and_shrinkage(): + # Real maintainer scenario: delete 2 MiB, add 5 files of 450 KiB (+2,250 KiB) + # Net addition is +206,848 bytes, which is well below the 2.0 MiB limit + files = [ + ("evidence/old_large.bin", 0, -2 * 1024 * 1024), + ] + [ + (f"evidence/new_{i}.bin", 450 * 1024, 450 * 1024) + for i in range(5) + ] + violations, total = check.check_file_budgets( + files, per_file_limit=LIMIT_FILE, per_pr_limit=LIMIT_PR + ) + assert violations == [] + assert total == (5 * 450 * 1024) - (2 * 1024 * 1024) + assert total == 206848 + + +def test_non_evidence_files_not_counted_in_evidence_budget(): + files = [ + ("src/inferrsi/engine.py", 100 * 1024, 100 * 1024), + ("tests/test_large.py", 200 * 1024, 200 * 1024), + ] + violations, total = check.check_file_budgets( + files, per_file_limit=LIMIT_FILE, per_pr_limit=LIMIT_PR + ) + assert violations == [] + assert total == 0 + + +def test_allowlist_bypasses_budget(): + path = "evidence/historical/grandfathered_raw.tar.gz" + files = [(path, 10 * 1024 * 1024, 10 * 1024 * 1024)] + violations, total = check.check_file_budgets( + files, + per_file_limit=LIMIT_FILE, + per_pr_limit=LIMIT_PR, + allowlist={path}, + ) + assert violations == [] + assert total == 0 + + +def test_validate_external_asset_manifest_valid(tmp_path: Path): + manifest = tmp_path / "inventory.json" + dummy_sha = "e" * 64 + valid_url = ( + "https://github.com/InferMatrixLab/InferRSI/releases/download/v0.3.0/traces.tar.gz" + ) + data = { + "version": 1, + "bundles": { + "qwen3-omni-two-gpu": { + "external_assets": [ + { + "url": valid_url, + "sha256": dummy_sha, + "bytes": 48920000, + "retention_owner": "Dong1017", + } + ] + } + }, + } + manifest.write_text(json.dumps(data), encoding="utf-8") + errors = check.validate_external_asset_manifest(manifest) + assert errors == [] + + +def test_validate_external_asset_manifest_rejects_tag_view_url(tmp_path: Path): + manifest = tmp_path / "inventory_tag.json" + data = { + "version": 1, + "bundles": { + "tag_bundle": { + "external_assets": [ + { + "url": "https://github.com/InferMatrixLab/InferRSI/releases/tag/v0.3.0", + "sha256": "a" * 64, + "bytes": 1000, + "retention_owner": "Dong1017", + } + ] + } + }, + } + manifest.write_text(json.dumps(data), encoding="utf-8") + errors = check.validate_external_asset_manifest(manifest) + assert len(errors) == 1 + assert "releases/download" in errors[0] + assert "format check" in errors[0] + + +def test_validate_external_asset_manifest_rejects_non_list_external_assets(tmp_path: Path): + manifest = tmp_path / "inventory_dict_assets.json" + data = { + "version": 1, + "bundles": { + "bad_bundle": { + "external_assets": { + "url": "https://github.com/InferMatrixLab/InferRSI/releases/download/v1/a.bin" + } + } + }, + } + manifest.write_text(json.dumps(data), encoding="utf-8") + errors = check.validate_external_asset_manifest(manifest) + assert len(errors) == 1 + assert "must be a list of asset objects" in errors[0] + + +def test_validate_external_asset_manifest_rejects_non_dict_bundles(tmp_path: Path): + manifest = tmp_path / "inventory_bad_bundles.json" + data = {"version": 1, "bundles": "not_a_dictionary"} + manifest.write_text(json.dumps(data), encoding="utf-8") + errors = check.validate_external_asset_manifest(manifest) + assert len(errors) == 1 + assert "'bundles' must be a JSON object" in errors[0] + + +def test_validate_external_asset_manifest_rejects_null_fields(tmp_path: Path): + manifest = tmp_path / "inventory_null_fields.json" + data = { + "version": 1, + "bundles": { + "null_bundle": { + "external_assets": [ + { + "url": None, + "sha256": None, + "bytes": None, + "retention_owner": None, + } + ] + } + }, + } + manifest.write_text(json.dumps(data), encoding="utf-8") + errors = check.validate_external_asset_manifest(manifest) + assert len(errors) == 4 + assert any("null required field 'url'" in e for e in errors) + assert any("null required field 'sha256'" in e for e in errors) + assert any("null required field 'bytes'" in e for e in errors) + assert any("null required field 'retention_owner'" in e for e in errors) + + +def test_invalid_base_ref_raises_error(tmp_path: Path): + with pytest.raises(ValueError, match="cannot be resolved"): + check.get_changed_files_with_net_sizes("non_existent_ref_12345", repo_root=tmp_path) + + +def test_git_rename_and_deletion_regression(tmp_path: Path): + """End-to-end Git test verifying rename, move in/out, delete net delta and non-ASCII paths.""" + repo = tmp_path / "test_repo" + repo.mkdir() + + def git(*args: str): + subprocess.run(["git", *args], cwd=repo, capture_output=True, check=True) + + git("init", "-b", "main") + git("config", "user.name", "Tester") + git("config", "user.email", "tester@example.com") + + # Initial base commit + ev_dir = repo / "evidence" + ev_dir.mkdir() + (ev_dir / "old_600k.bin").write_bytes(b"A" * (600 * 1024)) + (ev_dir / "to_delete_2m.bin").write_bytes(b"B" * (2 * 1024 * 1024)) + (ev_dir / "to_move_out.bin").write_bytes(b"D" * (50 * 1024)) + outside_dir = repo / "outside" + outside_dir.mkdir() + (outside_dir / "external.bin").write_bytes(b"C" * (100 * 1024)) + git("add", ".") + git("commit", "-m", "base commit") + + # Make changes in HEAD: + # 1. Rename 600 KiB file inside evidence/ -> should trigger per-file limit on dst! + git("mv", "evidence/old_600k.bin", "evidence/new_600k.bin") + + # 2. Delete 2 MiB file in evidence/ -> should yield -2 MiB delta + (ev_dir / "to_delete_2m.bin").unlink() + git("rm", "evidence/to_delete_2m.bin") + + # 3. Add non-ASCII file in evidence/ + (ev_dir / "é_trace.json").write_text('{"status": "ok"}', encoding="utf-8") + + # 4. Move outside file into evidence/ -> +100 KiB net addition + git("mv", "outside/external.bin", "evidence/external.bin") + + # 5. Move file OUT of evidence/ -> -50 KiB net addition + git("mv", "evidence/to_move_out.bin", "outside/moved_out.bin") + + git("add", ".") + git("commit", "-m", "head commit") + + records = check.get_changed_files_with_net_sizes("main~1", repo_root=repo) + paths = {r[0] for r in records} + assert "evidence/new_600k.bin" in paths + assert "evidence/é_trace.json" in paths + assert "evidence/external.bin" in paths + assert "evidence/to_move_out.bin" in paths + + # Move out of evidence must record negative delta + move_out_record = next(r for r in records if r[0] == "evidence/to_move_out.bin") + assert move_out_record[1] == 0 + assert move_out_record[2] == -(50 * 1024) + + violations, net_bytes = check.check_file_budgets(records) + # Per-file violation must catch evidence/new_600k.bin + assert any("evidence/new_600k.bin" in v and "exceeds per-file limit" in v for v in violations) + # Total net evidence is negative (~ -1.95 MB), so PR net limit is not violated + assert net_bytes < 0 + assert not any("per-PR net limit" in v for v in violations) + + +def test_git_symlink_not_measured_as_target_size(tmp_path: Path): + """Verify symlink blob is measured by its link target text length, not followed.""" + repo = tmp_path / "symlink_repo" + repo.mkdir() + + def git(*args: str, input_bytes: bytes | None = None): + return subprocess.run( + ["git", *args], + cwd=repo, + input=input_bytes, + capture_output=True, + check=True, + ) + + git("init", "-b", "main") + git("config", "user.name", "Tester") + git("config", "user.email", "tester@example.com") + + # Create 600 KiB file outside evidence + outside_dir = repo / "outside" + outside_dir.mkdir() + (outside_dir / "target.bin").write_bytes(b"X" * (600 * 1024)) + git("add", ".") + git("commit", "-m", "base commit") + + # Create git symlink object pointing to target.bin + blob_sha = git( + "hash-object", "-w", "--stdin", input_bytes=b"outside/target.bin" + ).stdout.decode().strip() + git("update-index", "--add", "--cacheinfo", "120000", blob_sha, "evidence/symlink.bin") + git("commit", "-m", "add symlink") + + records = check.get_changed_files_with_net_sizes("main~1", repo_root=repo) + symlink_record = next(r for r in records if r[0] == "evidence/symlink.bin") + # Blob size of 'outside/target.bin' is 18 bytes, NOT 600 KiB + assert symlink_record[1] == len("outside/target.bin") + violations, _ = check.check_file_budgets(records) + assert violations == [] + + +def test_git_empty_blob_with_dirty_worktree_not_misreported(tmp_path: Path): + """Verify empty blob committed in HEAD is not inflated by dirty uncommitted worktree.""" + repo = tmp_path / "empty_blob_repo" + repo.mkdir() + + def git(*args: str): + subprocess.run(["git", *args], cwd=repo, capture_output=True, check=True) + + git("init", "-b", "main") + git("config", "user.name", "Tester") + git("config", "user.email", "tester@example.com") + (repo / "README.md").write_text("# Base") + git("add", ".") + git("commit", "-m", "base") + + # In HEAD commit, commit an empty file in evidence/ + ev_dir = repo / "evidence" + ev_dir.mkdir() + empty_file = ev_dir / "empty.bin" + empty_file.write_bytes(b"") + git("add", ".") + git("commit", "-m", "add empty file") + + # Now write 600 KiB dirty worktree content without committing + empty_file.write_bytes(b"Z" * (600 * 1024)) + + records = check.get_changed_files_with_net_sizes("main~1", repo_root=repo) + empty_record = next(r for r in records if r[0] == "evidence/empty.bin") + # Committed blob in HEAD is 0 bytes, not 600 KiB + assert empty_record[1] == 0 + assert empty_record[2] == 0 + + violations, net_bytes = check.check_file_budgets(records) + assert violations == [] + assert net_bytes == 0 + + +def test_local_mode_untracked_files_expanded(tmp_path: Path): + """Verify local mode without base ref expands untracked nested directories.""" + repo = tmp_path / "local_repo" + repo.mkdir() + + def git(*args: str): + subprocess.run(["git", *args], cwd=repo, capture_output=True, check=True) + + git("init", "-b", "main") + git("config", "user.name", "Tester") + git("config", "user.email", "tester@example.com") + (repo / "README.md").write_text("# Repo") + git("add", ".") + git("commit", "-m", "init") + + # Add untracked deep nested file exceeding per-file limit + deep_dir = repo / "evidence" / "nested" / "run_01" + deep_dir.mkdir(parents=True) + (deep_dir / "large_raw.bin").write_bytes(b"Z" * (600 * 1024)) + + records = check.get_changed_files_with_net_sizes(base_ref=None, repo_root=repo) + matching = [r for r in records if "evidence/nested/run_01/large_raw.bin" in r[0]] + assert len(matching) == 1 + assert matching[0][1] == 600 * 1024 + + violations, _ = check.check_file_budgets(records) + assert any("large_raw.bin" in v and "exceeds per-file limit" in v for v in violations) + + +def test_main_negative_case_fails_without_report_only(tmp_path: Path, monkeypatch, capsys): + """Negative case: candidate budget violation without --report-only exits with code 1.""" + monkeypatch.setattr(check, "ROOT", tmp_path) + ev_dir = tmp_path / "evidence" + ev_dir.mkdir() + ev_file = ev_dir / "large_raw.bin" + ev_file.write_bytes(b"A" * (600 * 1024)) + + monkeypatch.setattr( + "sys.argv", + [ + "check_evidence_budget.py", + "--files", + str(ev_file), + "--max-file-bytes", + str(512 * 1024), + "--inventory", + "nonexistent_inv.json", + ], + ) + rc = check.main() + captured = capsys.readouterr() + assert rc == 1 + assert "exceeds per-file limit" in captured.out + assert "Evidence byte budget (report-only, #273 R6) exceeded" in captured.out + assert "Running in report-only mode" not in captured.out + + +def test_main_report_only_mode_warns_and_exits_zero(tmp_path: Path, monkeypatch, capsys): + """Report-only mode: candidate budget violation warns but exits 0 pending #273 acceptance.""" + monkeypatch.setattr(check, "ROOT", tmp_path) + ev_dir = tmp_path / "evidence" + ev_dir.mkdir() + ev_file = ev_dir / "candidate_large.bin" + ev_file.write_bytes(b"B" * (600 * 1024)) + + monkeypatch.setattr( + "sys.argv", + [ + "check_evidence_budget.py", + "--files", + str(ev_file), + "--max-file-bytes", + str(512 * 1024), + "--report-only", + "--inventory", + "nonexistent_inv.json", + ], + ) + rc = check.main() + captured = capsys.readouterr() + assert rc == 0 + assert "exceeds per-file limit" in captured.out + assert "Running in report-only mode" in captured.out + assert "Refs #273" in captured.out + + +def test_validate_external_asset_manifest_explicit_format_check_diagnostic(tmp_path: Path): + """External asset manifest URL diagnostic explicitly identifies validation as a format check.""" + manifest = tmp_path / "invalid_manifest.json" + data = { + "bundles": { + "test_bundle": { + "external_assets": [ + { + "url": "https://example.com/asset.zip", + "sha256": "0" * 64, + "bytes": 1024, + "retention_owner": "maintainer", + } + ] + } + } + } + manifest.write_text(json.dumps(data), encoding="utf-8") + errors = check.validate_external_asset_manifest(manifest) + assert len(errors) == 1 + assert "url format check failed" in errors[0] + assert "format check only; does not verify downloadability or remote hash" in errors[0] + + +def test_main_cli_help_mentions_format_check_and_report_only(monkeypatch, capsys): + """CLI help output explicitly mentions format check and --report-only mode.""" + monkeypatch.setattr("sys.argv", ["check_evidence_budget.py", "--help"]) + with pytest.raises(SystemExit) as exc: + check.main() + assert exc.value.code == 0 + captured = capsys.readouterr() + assert "format check" in captured.out + assert "--report-only" in captured.out + + +def test_git_raw_posix_backslash_top_level_path_not_treated_as_evidence(tmp_path: Path): + """A top-level file literally named 'evidence\\big.bin' must not count as evidence/.""" + repo = tmp_path / "backslash_repo" + repo.mkdir() + + def git(*args: str, input_bytes: bytes | None = None): + return subprocess.run( + ["git", *args], + cwd=repo, + input=input_bytes, + capture_output=True, + check=True, + ) + + git("init", "-b", "main") + git("config", "user.name", "Tester") + git("config", "user.email", "tester@example.com") + (repo / "README.md").write_text("# Repo") + git("add", ".") + git("commit", "-m", "init") + + # Base commit SHA + base_commit = git("rev-parse", "HEAD").stdout.decode().strip() + + # Create a 600 KiB blob in the git object database + blob_sha = git( + "hash-object", "-w", "--stdin", input_bytes=b"Z" * (600 * 1024) + ).stdout.decode().strip() + + # Use git mktree to construct a POSIX tree with top-level 'evidence\\raw_top_level.bin' + tree_input = f"100644 blob {blob_sha}\tevidence\\raw_top_level.bin\n".encode("utf-8") + tree_sha = git("mktree", input_bytes=tree_input).stdout.decode().strip() + + # Commit tree with base_commit as parent + head_commit = git( + "commit-tree", tree_sha, "-p", base_commit, "-m", "add top-level backslash file" + ).stdout.decode().strip() + + # Point HEAD / branch to head_commit + git("update-ref", "refs/heads/main", head_commit) + + records = check.get_changed_files_with_net_sizes(base_commit, repo_root=repo) + matching = [r for r in records if "evidence\\raw_top_level.bin" in r[0]] + assert len(matching) == 1 + rel_path, cur_size, _ = matching[0] + assert rel_path == "evidence\\raw_top_level.bin" + assert cur_size == 0 # Filtered before object-size read because not under evidence/ + + # Top-level file with literal backslash must NOT count as evidence/ directory + violations, net_bytes = check.check_file_budgets(records) + assert violations == [] + assert net_bytes == 0 + + +def test_main_report_only_mode_does_not_hide_manifest_errors(tmp_path: Path, monkeypatch, capsys): + """Report-only mode applies to candidate budgets; manifest schema/format errors still fail.""" + monkeypatch.setattr(check, "ROOT", tmp_path) + ev_dir = tmp_path / "evidence" + ev_dir.mkdir() + ev_file = ev_dir / "valid.bin" + ev_file.write_bytes(b"A" * 1024) + + # Invalid inventory manifest with invalid URL pattern and invalid sha256 + inv_file = tmp_path / "tools" / "evidence_inventory.json" + inv_file.parent.mkdir(parents=True) + inv_file.write_text( + json.dumps({ + "version": 1, + "bundles": { + "bad_bundle": { + "external_assets": [ + { + "url": "https://invalid.com/not_a_github_release", + "sha256": "bad_sha", + "bytes": 1024, + "retention_owner": "owner", + } + ] + } + }, + }), + encoding="utf-8", + ) + + monkeypatch.setattr( + "sys.argv", + [ + "check_evidence_budget.py", + "--files", + str(ev_file), + "--report-only", + "--inventory", + "tools/evidence_inventory.json", + ], + ) + rc = check.main() + captured = capsys.readouterr() + # Must exit 1 despite --report-only because manifest errors are enforced + assert rc == 1 + assert "url format check failed" in captured.err + assert "invalid sha256 hex digest" in captured.err + + +def test_unrelated_gitlink_operations_do_not_fail_budget_checker(tmp_path: Path): + """Verify gitlink (mode 160000) add, update, and delete do not raise or count as evidence.""" + repo = tmp_path / "gitlink_repo" + repo.mkdir() + + def git(*args: str): + res = subprocess.run(["git", *args], cwd=repo, capture_output=True, check=True) + return res.stdout.decode("utf-8").strip() + + git("init", "-b", "main") + git("config", "user.name", "Tester") + git("config", "user.email", "tester@example.com") + (repo / "README.md").write_text("# Repo") + git("add", ".") + git("commit", "-m", "init") + base = git("rev-parse", "HEAD") + + # 1. Add unrelated gitlink vendor/example with absent commit SHA + fake_sha1 = "1234567890abcdef1234567890abcdef12345678" + git("update-index", "--add", "--cacheinfo", "160000", fake_sha1, "vendor/example") + git("commit", "-m", "add gitlink") + + records_add = check.get_changed_files_with_net_sizes(base, repo_root=repo) + assert any(r[0] == "vendor/example" and r[1] == 0 and r[2] == 0 for r in records_add) + violations_add, total_add = check.check_file_budgets(records_add) + assert violations_add == [] + assert total_add == 0 + + # 2. Update unrelated gitlink to another absent commit SHA + fake_sha2 = "9876543210fedcba9876543210fedcba98765432" + git("update-index", "--add", "--cacheinfo", "160000", fake_sha2, "vendor/example") + git("commit", "-m", "update gitlink") + + records_upd = check.get_changed_files_with_net_sizes("HEAD~1", repo_root=repo) + assert any(r[0] == "vendor/example" and r[1] == 0 and r[2] == 0 for r in records_upd) + violations_upd, total_upd = check.check_file_budgets(records_upd) + assert violations_upd == [] + assert total_upd == 0 + + # 3. Delete unrelated gitlink + git("update-index", "--force-remove", "vendor/example") + git("commit", "-m", "delete gitlink") + + records_del = check.get_changed_files_with_net_sizes("HEAD~1", repo_root=repo) + assert any(r[0] == "vendor/example" and r[1] == 0 and r[2] == 0 for r in records_del) + violations_del, total_del = check.check_file_budgets(records_del) + assert violations_del == [] + assert total_del == 0 + + +def test_missing_blob_under_evidence_still_raises_value_error(tmp_path: Path): + """Verify genuinely missing blob under evidence/ raises ValueError on diff sizing.""" + repo = tmp_path / "missing_blob_repo" + repo.mkdir() + + def git(*args: str): + res = subprocess.run(["git", *args], cwd=repo, capture_output=True, check=True) + return res.stdout.decode("utf-8").strip() + + git("init", "-b", "main") + git("config", "user.name", "Tester") + git("config", "user.email", "tester@example.com") + (repo / "README.md").write_text("# Repo") + git("add", ".") + git("commit", "-m", "init") + base = git("rev-parse", "HEAD") + + # Add missing blob under evidence/ with mode 100644 + fake_blob = "00112233445566778899aabbccddeeff00112233" + git("update-index", "--add", "--cacheinfo", "100644", fake_blob, "evidence/missing.bin") + tree_sha = git("write-tree", "--missing-ok") + head = git("commit-tree", tree_sha, "-p", base, "-m", "missing blob in evidence") + git("update-ref", "refs/heads/main", head) + + with pytest.raises(ValueError, match="Failed to read size for git object"): + check.get_changed_files_with_net_sizes(base, repo_root=repo) + + + diff --git a/tools/check_evidence_budget.py b/tools/check_evidence_budget.py new file mode 100644 index 00000000..acbb8214 --- /dev/null +++ b/tools/check_evidence_budget.py @@ -0,0 +1,433 @@ +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# Report-only R6 Byte Budgets (adopted in #273 R6; enforcement pending 2026-12-02 review) +# Per-file budget: 512 KiB for files tracked in evidence/ +PER_FILE_MAX_BYTES = 512 * 1024 +# Per-PR net evidence addition budget: 2.0 MiB +PER_PR_EVIDENCE_MAX_BYTES = 2 * 1024 * 1024 + +RELEASE_ASSET_URL_PATTERN = re.compile( + r"^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/releases/download/[^/]+/[^/]+$" +) + +ADVICE = ( + "Evidence byte budget (report-only, #273 R6) exceeded. Large raw artifacts should be stored " + "outside Git as GitHub Release assets, recording their downloadable asset URL, " + "SHA-256 digest, byte count, and retention owner in tools/evidence_inventory.json. " + "See decision in Issue #273 (Remedy R6)." +) + + +def get_git_object_size(root: Path, obj_ref: str) -> int: + """Return byte size of a git blob object using git cat-file -s. + + Raises ValueError if git cat-file fails. + """ + if not obj_ref or set(obj_ref) == {"0"}: + return 0 + res = subprocess.run( + ["git", "cat-file", "-s", obj_ref], + cwd=root, + capture_output=True, + text=True, + ) + if res.returncode != 0: + raise ValueError( + f"Failed to read size for git object '{obj_ref}': {res.stderr.strip()}" + ) + try: + return int(res.stdout.strip()) + except ValueError as exc: + raise ValueError( + f"Invalid size output for git object '{obj_ref}': {res.stdout.strip()}" + ) from exc + + +def get_changed_files_with_net_sizes( + base_ref: str | None = None, + repo_root: Path | None = None, +) -> list[tuple[str, int, int]]: + """Return list of (rel_path, current_size_bytes, net_addition_bytes). + + If base_ref is supplied, uses git diff --raw -z against base_ref. + Sizes are obtained from git blob objects directly (preventing symlink + following, empty blob confusion, and worktree path confusion). + If base_ref fails or is missing, raises ValueError (no silent fallback in CI). + If base_ref is None, inspects local git status with -z and --untracked-files=all. + """ + root = repo_root or ROOT + results: list[tuple[str, int, int]] = [] + + if base_ref: + rev_check = subprocess.run( + ["git", "rev-parse", "--verify", base_ref], + cwd=root, + capture_output=True, + text=True, + ) + if rev_check.returncode != 0: + raise ValueError( + f"Base ref '{base_ref}' cannot be resolved. " + "Ensure target base ref is fetched before checking diff." + ) + + res = subprocess.run( + ["git", "diff", "--raw", "-z", f"{base_ref}...HEAD"], + cwd=root, + capture_output=True, + check=True, + ) + + tokens = res.stdout.split(b"\0") + i = 0 + while i < len(tokens): + tok = tokens[i] + if not tok: + i += 1 + continue + if not tok.startswith(b":"): + i += 1 + continue + + header_str = tok.decode("utf-8", errors="replace") + parts = header_str.split() + if len(parts) < 5: + i += 1 + continue + + old_mode = parts[0].lstrip(":") + new_mode = parts[1] + sha_src, sha_dst, status = parts[2], parts[3], parts[4] + + is_gitlink = old_mode == "160000" or new_mode == "160000" + + is_rename_or_copy = status.startswith(("R", "C")) + if is_rename_or_copy: + if i + 2 >= len(tokens): + break + src_path = tokens[i + 1].decode("utf-8", errors="replace") + dst_path = tokens[i + 2].decode("utf-8", errors="replace") + i += 3 + + src_is_ev = src_path.startswith("evidence/") + dst_is_ev = dst_path.startswith("evidence/") + + # Filter before reading object sizes; gitlinks never count as evidence bytes + src_size = ( + get_git_object_size(root, sha_src) + if (src_is_ev and old_mode != "160000") + else 0 + ) + dst_size = ( + get_git_object_size(root, sha_dst) + if (dst_is_ev and new_mode != "160000") + else 0 + ) + + if status.startswith("R"): + # Rename: src replaced by dst + if src_is_ev and dst_is_ev: + delta = dst_size - src_size + results.append((dst_path, dst_size, delta)) + elif not src_is_ev and dst_is_ev: + # Moved into evidence + results.append((dst_path, dst_size, dst_size)) + elif src_is_ev and not dst_is_ev: + # Moved out of evidence + results.append((src_path, 0, -src_size)) + else: + results.append((dst_path, 0, 0)) + else: + # Copy: src still exists, dst is added + delta = dst_size if dst_is_ev else 0 + results.append((dst_path, dst_size, delta)) + + else: + if i + 1 >= len(tokens): + break + path = tokens[i + 1].decode("utf-8", errors="replace") + i += 2 + + is_ev = path.startswith("evidence/") + + if not is_ev or is_gitlink: + # Unrelated or gitlink: do not read object sizes or count evidence bytes + results.append((path, 0, 0)) + elif status.startswith("D"): + src_size = get_git_object_size(root, sha_src) + results.append((path, 0, -src_size)) + elif status.startswith("A"): + cur_size = get_git_object_size(root, sha_dst) + results.append((path, cur_size, cur_size)) + elif status.startswith("M") or status.startswith("T"): + src_size = get_git_object_size(root, sha_src) + cur_size = get_git_object_size(root, sha_dst) + results.append((path, cur_size, cur_size - src_size)) + else: + cur_size = get_git_object_size(root, sha_dst) + results.append((path, cur_size, cur_size)) + + else: + # Local development fallback inspecting uncommitted changes with -z -uall + local_paths = get_local_git_changed_files(root) + for rel_path in local_paths: + full_path = root / rel_path + if full_path.is_file() or full_path.is_symlink(): + sz = full_path.lstat().st_size + results.append((rel_path, sz, sz)) + + return results + + +def get_local_git_changed_files(root: Path) -> list[str]: + """Inspect local git status with -z and --untracked-files=all.""" + try: + res = subprocess.run( + ["git", "status", "--porcelain", "-z", "--untracked-files=all"], + cwd=root, + capture_output=True, + check=True, + ) + tokens = res.stdout.split(b"\0") + paths: list[str] = [] + i = 0 + while i < len(tokens): + tok = tokens[i] + if not tok: + i += 1 + continue + status_code = tok[:2].decode("latin-1", errors="replace") + path = tok[3:].decode("utf-8", errors="replace") + if "R" in status_code: + # In git status -z for rename: new path is first, old path is second token + paths.append(path) + i += 2 + else: + if "D" not in status_code: + paths.append(path) + i += 1 + return paths + except (subprocess.SubprocessError, FileNotFoundError): + return [] + + +def check_file_budgets( + file_records: list[tuple[str, int, int]], + per_file_limit: int = PER_FILE_MAX_BYTES, + per_pr_limit: int = PER_PR_EVIDENCE_MAX_BYTES, + allowlist: set[str] | None = None, +) -> tuple[list[str], int]: + """Check file records (rel_path, current_size, net_addition) against budgets.""" + violations: list[str] = [] + total_net_evidence_bytes = 0 + allowed = allowlist or set() + + for rel_path, current_size, net_addition in file_records: + norm_path = rel_path + if norm_path in allowed: + continue + + if norm_path.startswith("evidence/"): + total_net_evidence_bytes += net_addition + if current_size > per_file_limit: + violations.append( + f"file {norm_path}: {current_size:,} bytes exceeds per-file limit " + f"of {per_file_limit:,} bytes" + ) + + if total_net_evidence_bytes > per_pr_limit: + sign = "+" if total_net_evidence_bytes >= 0 else "-" + violations.append( + f"evidence net addition: {sign}{abs(total_net_evidence_bytes):,} bytes " + f"exceeds per-PR net limit of {per_pr_limit:,} bytes" + ) + + return violations, total_net_evidence_bytes + + +def validate_external_asset_manifest(manifest_path: Path) -> list[str]: + """Verify external asset manifest syntax and fields conform to the schema. + + Validates schema structure, types, and performs a format check on GitHub Release download URLs. + This is a format check only and does not verify asset downloadability or remote hashes. + """ + errors: list[str] = [] + if not manifest_path.exists(): + return errors + + try: + data = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception as exc: + return [f"manifest error: failed to parse {manifest_path.name}: {exc}"] + + if not isinstance(data, dict): + return ["manifest error: root of manifest must be a JSON object"] + + if "bundles" not in data: + return ["manifest error: missing required key 'bundles'"] + + bundles = data.get("bundles") + if not isinstance(bundles, dict): + return ["manifest error: 'bundles' must be a JSON object"] + + sha_re = re.compile(r"^[0-9a-f]{64}$") + + for bundle_name, bundle in bundles.items(): + if not isinstance(bundle, dict): + errors.append(f"bundle {bundle_name}: expected JSON object") + continue + + if "external_assets" not in bundle: + continue + + external_assets = bundle["external_assets"] + if not isinstance(external_assets, list): + errors.append( + f"bundle {bundle_name}: 'external_assets' must be a list of asset objects" + ) + continue + + for idx, asset in enumerate(external_assets): + prefix = f"bundle {bundle_name} external_assets[{idx}]" + if not isinstance(asset, dict): + errors.append(f"{prefix}: expected JSON object") + continue + + # url format check + url = asset.get("url") + if url is None: + errors.append(f"{prefix}: missing or null required field 'url'") + elif not isinstance(url, str) or not RELEASE_ASSET_URL_PATTERN.match(url): + errors.append( + f"{prefix}: url format check failed: url must match pattern " + f"https://github.com///releases/download// " + f"(format check only; does not verify downloadability or remote hash)" + ) + + # sha256 check + sha = asset.get("sha256") + if sha is None: + errors.append(f"{prefix}: missing or null required field 'sha256'") + elif not isinstance(sha, str) or not sha_re.match(sha): + errors.append(f"{prefix}: invalid sha256 hex digest") + + # bytes check + byte_count = asset.get("bytes") + if byte_count is None: + errors.append(f"{prefix}: missing or null required field 'bytes'") + elif ( + not isinstance(byte_count, int) + or isinstance(byte_count, bool) + or byte_count <= 0 + ): + errors.append(f"{prefix}: bytes must be a positive integer") + + # retention_owner check + owner = asset.get("retention_owner") + if owner is None: + errors.append(f"{prefix}: missing or null required field 'retention_owner'") + elif not isinstance(owner, str) or not owner.strip(): + errors.append(f"{prefix}: retention_owner must be non-empty string") + + return errors + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check evidence byte budgets and external asset metadata (format check)." + ) + parser.add_argument("--base", help="Base git reference/SHA (e.g. origin/main or $BASE_SHA)") + parser.add_argument("--files", nargs="*", help="Explicit list of files to check") + parser.add_argument( + "--inventory", + default="tools/evidence_inventory.json", + help="Path to evidence inventory (performs format check only on asset metadata)", + ) + parser.add_argument( + "--max-file-bytes", type=int, default=PER_FILE_MAX_BYTES, help="Per-file max byte budget" + ) + parser.add_argument( + "--max-pr-bytes", + type=int, + default=PER_PR_EVIDENCE_MAX_BYTES, + help="Per-PR max byte budget", + ) + parser.add_argument( + "--report-only", + action="store_true", + help=( + "Report candidate budget violations as warnings without exiting with error code " + "(for evaluation while candidate thresholds are under discussion in #273)" + ), + ) + args = parser.parse_args() + + budget_violations: list[str] = [] + + try: + if args.files is not None: + file_list: list[tuple[str, int, int]] = [] + for p_str in args.files: + p = Path(p_str) + if not p.is_absolute(): + p = (ROOT / p) + if p.is_file() or p.is_symlink(): + try: + rel = p.relative_to(ROOT).as_posix() + except ValueError: + rel = p_str.replace("\\", "/") + sz = p.lstat().st_size + file_list.append((rel, sz, sz)) + diff_violations, _ = check_file_budgets( + file_list, args.max_file_bytes, args.max_pr_bytes + ) + budget_violations.extend(diff_violations) + else: + changed_records = get_changed_files_with_net_sizes(args.base) + diff_violations, _ = check_file_budgets( + changed_records, args.max_file_bytes, args.max_pr_bytes + ) + budget_violations.extend(diff_violations) + except Exception as exc: + print(f"evidence-budget error: {exc}", file=sys.stderr) + return 1 + + manifest_errors: list[str] = [] + inv_path = ROOT / args.inventory + if inv_path.exists(): + manifest_errors.extend(validate_external_asset_manifest(inv_path)) + + for item in budget_violations: + print(item) + + if budget_violations: + print(ADVICE) + if args.report_only: + print( + "[NOTICE] Running in report-only mode: candidate byte budget violations " + "reported above as warnings; check will not fail CI (Refs #273)." + ) + + for err in manifest_errors: + print(err, file=sys.stderr) + + if manifest_errors: + return 1 + + if budget_violations and not args.report_only: + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main())