diff --git a/CHANGELOG.md b/CHANGELOG.md index 749c1189..7d58f467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -588,6 +588,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 grepping the text report, so a reworded message cannot hide a broken dependency, and it refuses a `dist/` holding more than one wheel of a package (#575). +- A filtered audit is no longer reused as a full one (#542). The audit + cache records the audit's scope: framework, level, tag filter, and + whether any filter narrowed the control set. A filtered audit is cached + under its own key, and remediation reads only an unfiltered level 3 + audit of its framework, through the store the audit wrote to. Before, + `darnit audit --tags level=1` left a cache that remediation treated as + a full audit. Cache files from earlier versions are ignored and rebuilt. +- A filtered audit is no longer reused as a full one (#542). Only an + unfiltered audit is cached: a run narrowed by `--tags`, `--include`, + `--exclude` or a profile does not write the audit cache. Remediation + reads only a level 3 audit of its framework, through the store the + audit wrote to, and expires it through that same store after it changes + something. Before, `darnit audit --tags level=1` left a cache that + remediation treated as a full audit, and with a configured + `[stores.cache]` a remediation left the old audit in place. Cache files + from earlier versions are ignored and rebuilt. - Installs without the development dependencies work again (`uv tool install` from a checkout, and any install from a built wheel). Two things broke them: `tree-sitter-language-pack` 1.6.3, the newest release the old diff --git a/docs/architecture/framework-design.md b/docs/architecture/framework-design.md index e09db85f..e7d8f8ff 100644 --- a/docs/architecture/framework-design.md +++ b/docs/architecture/framework-design.md @@ -1870,8 +1870,24 @@ Cache files are stored in the system temp directory (`$TMPDIR/darnit/ - **AND** controls with `status == "WARN"` SHALL NOT be remediated (WARN means automated verification was inconclusive, not that the control is non-compliant) - **AND** controls with `status == "PASS"` SHALL NOT be remediated +#### Requirement: The cache envelope records the audit's scope +- **WHEN** `write_audit_cache()` is called +- **THEN** the envelope SHALL record `framework` and `level` +- **AND** `run_sieve_audit()` SHALL NOT write the cache for a filtered audit, so a subset of the controls is never stored where a full audit is expected +- **AND** `run_sieve_audit()` SHALL treat an audit as filtered when it has tags, or when its caller passed `controls` without stating `cache_filtered`; a caller that passes the full control set SHALL state `cache_filtered=False` +- **AND** the cache key SHALL be derived in one place (`audit_cache_key()`), used by the writer, the reader and the invalidation + +#### Requirement: A cache read may constrain the scope it accepts +- **WHEN** `read_audit_cache()` is called with `expected_framework` or `expected_level` +- **THEN** a mismatch on any supplied expectation SHALL be treated as a cache miss +- **AND** a caller supplying none of them SHALL be unaffected +- **AND** an envelope whose `version` differs from `CACHE_VERSION` SHALL be a miss, since an older envelope may hold a filtered audit + #### Requirement: Remediation consumes cached audit results -- **WHEN** `remediate_audit_findings()` is called and `read_audit_cache()` returns valid cached results +- **WHEN** `remediate_audit_findings()` is called +- **THEN** it SHALL read through the store selected by `[stores.cache]`, the same store the audit writes to +- **AND** it SHALL require an unfiltered level-3 `openssf-baseline` cache, matching the audit it would otherwise run +- **WHEN** `read_audit_cache()` returns valid cached results - **THEN** it SHALL extract failed control IDs from the cached results (entries with `status == "FAIL"`) - **AND** it SHALL NOT run a redundant audit - **WHEN** `read_audit_cache()` returns `None` (cache miss) @@ -1880,9 +1896,9 @@ Cache files are stored in the system temp directory (`$TMPDIR/darnit/ #### Requirement: Post-remediation cache invalidation - **WHEN** `remediate_audit_findings()` completes with `dry_run=False` and at least one outcome changed something (section 15.4) -- **THEN** it SHALL call `invalidate_audit_cache(local_path)` +- **THEN** it SHALL expire the cached full audit through the same store and key the cache read uses (`invalidate_full_audit_cache(local_path, framework_name)`), so that a configured `[stores.cache]` is invalidated and not only the default store - **WHEN** `remediate_audit_findings()` completes with `dry_run=True` -- **THEN** it SHALL NOT call `invalidate_audit_cache()` +- **THEN** it SHALL NOT invalidate the cache ### 10.5 Framework Contains No Implementation-Specific Code diff --git a/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py b/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py index 838d0f38..b868077f 100644 --- a/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py +++ b/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py @@ -1117,8 +1117,13 @@ def run_remediation( error: str | None = None try: - from darnit.core.audit_cache import read_audit_cache - cache = read_audit_cache(local_path) + # Issue #542: remediation acts on a full level-3 audit of this + # framework, which is what it runs on a miss. A level-1, filtered or + # other-framework cache is a subset, so it must miss. The read goes + # through the store and key the audit writes to. + from darnit.tools.audit import read_full_audit_cache + + cache = read_full_audit_cache(local_path, "openssf-baseline", level=3) except Exception as exc: logger.warning(f"Audit cache read failed: {exc}") cache = None @@ -1279,9 +1284,9 @@ def run_remediation( outcomes = _outcomes(results, lambda ids: _recheck(ids, local_path, owner, repo, target)) if any(_written(r) or _applied_change_sets(r) for r in results): try: - from darnit.core import audit_cache + from darnit.tools.audit import invalidate_full_audit_cache - audit_cache.invalidate_audit_cache(local_path) + invalidate_full_audit_cache(local_path, "openssf-baseline") except Exception as exc: logger.warning(f"Failed to invalidate audit cache: {exc}") diff --git a/packages/darnit-baseline/src/darnit_baseline/tools.py b/packages/darnit-baseline/src/darnit_baseline/tools.py index 4f68a031..8cd42360 100644 --- a/packages/darnit-baseline/src/darnit_baseline/tools.py +++ b/packages/darnit-baseline/src/darnit_baseline/tools.py @@ -226,6 +226,9 @@ def audit_openssf_baseline( level=level, controls=controls, tags=tags_list, + # Issue #542: a profile narrows `controls` above. Tags are + # recorded by run_sieve_audit itself. + cache_filtered=bool(profile), stop_on_llm=True, framework_name="openssf-baseline", operator_config=operator_config, @@ -504,9 +507,9 @@ def create_security_policy( written = [c.path for c in outcome.file_changes if c.changes] if written: try: - from darnit.core import audit_cache + from darnit.tools.audit import invalidate_full_audit_cache - audit_cache.invalidate_audit_cache(str(repo_path)) + invalidate_full_audit_cache(str(repo_path), "openssf-baseline") except Exception as exc: # noqa: BLE001 from darnit.core.logging import get_logger diff --git a/packages/darnit/src/darnit/cli.py b/packages/darnit/src/darnit/cli.py index d056c9d4..b9732d16 100644 --- a/packages/darnit/src/darnit/cli.py +++ b/packages/darnit/src/darnit/cli.py @@ -312,6 +312,8 @@ def cmd_audit(args: argparse.Namespace) -> int: include_ids = set(args.include.split(",")) if args.include else None exclude_ids = set(args.exclude.split(",")) if args.exclude else set() + loaded_count = len(controls) + # Apply filters controls = filter_controls(controls, filters, include_ids, exclude_ids) @@ -334,6 +336,10 @@ def cmd_audit(args: argparse.Namespace) -> int: default_branch=default_branch, level=3, controls=controls, + # Issue #542: the CLI narrows `controls` itself, so it says whether + # it did. Comparing the counts covers every filter, present and + # future, without listing the flags. A filtered audit is not cached. + cache_filtered=len(controls) < loaded_count, stop_on_llm=True, # Issue #427: the framework name has to reach the audit driver, not # just the control loader above. Without it the driver cannot diff --git a/packages/darnit/src/darnit/core/audit_cache.py b/packages/darnit/src/darnit/core/audit_cache.py index 9e587025..88dd70e4 100644 --- a/packages/darnit/src/darnit/core/audit_cache.py +++ b/packages/darnit/src/darnit/core/audit_cache.py @@ -55,7 +55,7 @@ logger = get_logger("core.audit_cache") CACHE_FILENAME = "audit-cache.json" -CACHE_VERSION = 1 +CACHE_VERSION = 2 # Invalidation via write-expired-envelope (feature 035 clarify Q3). # The AuditCacheStore Protocol has no delete(key) method; instead, @@ -232,6 +232,8 @@ def read_audit_cache( local_path: str, ttl_seconds: int = 3600, *, + expected_framework: str | None = None, + expected_level: int | None = None, store: AuditCacheStore | None = None, cache_key: str | None = None, ) -> dict[str, Any] | None: @@ -266,10 +268,33 @@ def read_audit_cache( # Version check version = data.get("version") - if not isinstance(version, int) or version > CACHE_VERSION: + # Issue #542 review: an exact match, not an upper bound. A v1 envelope + # may hold a filtered audit under the full-audit key, so reading it would + # present a subset as the full audit. Older caches miss and are re-audited. + if not isinstance(version, int) or version != CACHE_VERSION: logger.debug("Unknown audit cache version: %s", version) return None + # Scope: the cached audit must cover what the caller is about to use it + # for (issue #542). A level-1 or other-framework run leaves a cache that + # is not the audit the caller asked for; reusing it would make a consumer + # act on results that were never produced. Filtered audits are never + # written, see run_sieve_audit. + if expected_framework is not None and data.get("framework") != expected_framework: + logger.debug( + "Audit cache scope mismatch: framework %r != expected %r", + data.get("framework"), + expected_framework, + ) + return None + if expected_level is not None and data.get("level") != expected_level: + logger.debug( + "Audit cache scope mismatch: level %r != expected %r", + data.get("level"), + expected_level, + ) + return None + # Staleness: TTL expiry timestamp_str = data.get("timestamp") if isinstance(timestamp_str, str): diff --git a/packages/darnit/src/darnit/server/tools/builtin_audit.py b/packages/darnit/src/darnit/server/tools/builtin_audit.py index 833af0a3..8212ebd2 100644 --- a/packages/darnit/src/darnit/server/tools/builtin_audit.py +++ b/packages/darnit/src/darnit/server/tools/builtin_audit.py @@ -133,6 +133,9 @@ async def builtin_audit( level=level, controls=all_controls, tags=tags_list, + # Issue #542: all_controls is the framework's full set. Tags are + # recorded by run_sieve_audit itself. + cache_filtered=False, stop_on_llm=True, framework_name=_framework_name, operator_config=operator_config, diff --git a/packages/darnit/src/darnit/tools/audit.py b/packages/darnit/src/darnit/tools/audit.py index ed76c33b..74aa0962 100644 --- a/packages/darnit/src/darnit/tools/audit.py +++ b/packages/darnit/src/darnit/tools/audit.py @@ -167,6 +167,105 @@ def _load_merged_stores(framework_name: str | None, operator: "OperatorConfig | return getattr(effective, "stores", None) +def audit_cache_key(local_path: str, stores_config: Any) -> str: + """Return the key the full audit of ``local_path`` is cached under. + + The one derivation for the writer (:func:`run_sieve_audit`), the reader + and the invalidation. The default store's root already encodes the repository, so the + key is fixed; a configured store may be shared across repositories, so the + key encodes the path. Only unfiltered audits are cached (issue #542), so + there is one entry per repository. + """ + if stores_config is None or stores_config.cache is None: + return "audit-cache" + return hashlib.sha256(str(Path(local_path).resolve()).encode()).hexdigest()[:16] + + +def _is_filtered(tags: list[str] | None, controls: list | None, cache_filtered: bool | None) -> bool: + """Return whether this audit covers less than the full control set (issue #542). + + Tags always mean filtered. Otherwise the caller's word decides. A caller + that passed its own ``controls`` and did not say is treated as filtered: + this function cannot tell a narrowed list from the full set, and a subset + read back as a full audit makes remediation act on results that were never + produced. + """ + if tags: + return True + if cache_filtered is None: + return controls is not None + return cache_filtered + + +def _full_audit_cache_location( + local_path: str, + framework_name: str, + operator_config: "LoadedOperatorConfig | None", +) -> tuple[Any, str]: + """Return the (stores bundle, key) the full audit of ``local_path`` is cached at. + + The store and key :func:`run_sieve_audit` writes to, so an operator's + ``[stores.cache]`` is honoured by the read and by the invalidation alike. + The caller closes the bundle. + """ + from darnit.stores.selection import resolve_stores + + if operator_config is None: + from darnit.config.operator.loader import resolve_operator_config + + operator_config = resolve_operator_config(local_path) + stores_config = _load_merged_stores(framework_name, operator_config.config) + stores_bundle = resolve_stores(stores_config, repo_path=Path(local_path)) + return stores_bundle, audit_cache_key(local_path, stores_config) + + +def read_full_audit_cache( + local_path: str, + framework_name: str, + level: int, + operator_config: "LoadedOperatorConfig | None" = None, +) -> dict[str, Any] | None: + """Return the cached full audit of ``framework_name`` at ``level``, or None. + + A cache from another framework or another level is a miss. A filtered + audit is never cached, so it cannot be returned here. + """ + from darnit.core.audit_cache import read_audit_cache + + stores_bundle, cache_key = _full_audit_cache_location(local_path, framework_name, operator_config) + try: + return read_audit_cache( + local_path, + expected_framework=framework_name, + expected_level=level, + store=stores_bundle.cache, + cache_key=cache_key, + ) + finally: + stores_bundle.close_all() + + +def invalidate_full_audit_cache( + local_path: str, + framework_name: str, + operator_config: "LoadedOperatorConfig | None" = None, +) -> None: + """Expire the cached full audit, in the place :func:`read_full_audit_cache` reads it. + + Remediation calls this after it changed something. With a configured + ``[stores.cache]``, expiring the default store's entry would leave the + configured one in place, and the next remediation would act on the audit + from before the change. + """ + from darnit.core.audit_cache import invalidate_audit_cache + + stores_bundle, cache_key = _full_audit_cache_location(local_path, framework_name, operator_config) + try: + invalidate_audit_cache(local_path, store=stores_bundle.cache, cache_key=cache_key) + finally: + stores_bundle.close_all() + + def _load_merged_mcp_servers(framework_name: str | None, operator: "OperatorConfig | None" = None) -> dict[str, Any]: """Return the merged ``mcp_servers`` allowlist for this audit run. @@ -731,6 +830,7 @@ def run_sieve_audit( *, controls: list | None = None, tags: list[str] | None = None, + cache_filtered: bool | None = None, evaluate_claims: bool = True, stop_on_llm: bool = True, framework_name: str | None = None, @@ -775,6 +875,9 @@ def run_sieve_audit( write_cache: Write the results to the audit cache. A subset run, such as remediation's re-check (feature 043), passes False so it does not replace the full-audit cache. + cache_filtered: Whether ``controls`` is narrower than the framework's + full set. A filtered audit is not cached (issue #542). When unset, + a caller that passes ``controls`` is treated as filtered. Returns: Tuple of (results, summary) where results is a list of check result @@ -1014,10 +1117,10 @@ def run_sieve_audit( # already encodes repo identity (uses fixed "audit-cache"); configured # store's root is operator-picked and may be shared across repos, so # the key must encode repo identity. - if stores_config is None or stores_config.cache is None: - cache_key = "audit-cache" - else: - cache_key = hashlib.sha256(str(Path(local_path).resolve()).encode()).hexdigest()[:16] + # Issue #542: only an unfiltered audit is cached. A subset stored here + # would be read back by remediation as if it were the full audit. + cache_filtered = _is_filtered(tags, controls, cache_filtered) + cache_key = audit_cache_key(local_path, stores_config) from darnit.core.audit_cache import write_audit_cache @@ -1025,7 +1128,7 @@ def run_sieve_audit( # internally. The enclosing try/except is defensive belt-and-braces # against any unexpected exception path (e.g. TypeError from a # future signature change). - if write_cache: + if write_cache and not cache_filtered: try: write_audit_cache( local_path, diff --git a/tests/darnit/core/test_audit_cache.py b/tests/darnit/core/test_audit_cache.py index d93ae306..02571ed8 100644 --- a/tests/darnit/core/test_audit_cache.py +++ b/tests/darnit/core/test_audit_cache.py @@ -475,3 +475,156 @@ def worker(thread_id): final_data = read_audit_cache(repo_path_str) assert final_data is not None assert final_data["version"] == CACHE_VERSION + + +class TestScopeMismatch: + """Issue #542: a cache from a narrower audit must not satisfy a wider read.""" + + def test_framework_mismatch_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 3, "openssf-baseline") + assert read_audit_cache(str(temp_git_repo), expected_framework="openssf-baseline") is not None + assert read_audit_cache(str(temp_git_repo), expected_framework="amber") is None + + def test_level_mismatch_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 1, "openssf-baseline") + assert read_audit_cache(str(temp_git_repo), expected_level=1) is not None + assert read_audit_cache(str(temp_git_repo), expected_level=3) is None + + def test_unscoped_read_still_hits(self, temp_git_repo, sample_results, sample_summary): + """Existing callers that pass no expectations are unaffected.""" + write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 1, "amber") + assert read_audit_cache(str(temp_git_repo)) is not None + + +class TestCacheVersion: + """Review on #549: an envelope written before the scope fix must miss. + + Version 1 envelopes could hold a filtered audit under the full-audit key. + """ + + def test_old_version_envelope_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 3, "openssf-baseline") + cache_path = _get_cache_dir(str(temp_git_repo)) / CACHE_FILENAME + data = json.loads(cache_path.read_text()) + data["version"] = 1 + cache_path.write_text(json.dumps(data)) + assert read_audit_cache(str(temp_git_repo)) is None + + +class TestFilteredDecision: + """Whether run_sieve_audit treats an audit as filtered, and so does not cache it (#542).""" + + @pytest.mark.parametrize( + ("tags", "controls", "cache_filtered", "expected"), + [ + (None, None, None, False), # darnit loads the controls: a full audit + (["domain=VM"], None, None, True), + (None, ["c"], None, True), # caller's own list, not declared: fail closed + (None, ["c"], False, False), # caller says it is the full set + (None, ["c"], True, True), # caller says it narrowed, e.g. a profile + (["level=1"], ["c"], False, True), # tags win over "not filtered" + ], + ) + def test_is_filtered(self, tags, controls, cache_filtered, expected): + from darnit.tools.audit import _is_filtered + + assert _is_filtered(tags, controls, cache_filtered) is expected + + def test_one_key_per_repository(self): + from darnit.tools.audit import audit_cache_key + + assert audit_cache_key("/repo", None) == "audit-cache" + + +def _write_full_audit(repo, results, summary, *, level=3, framework="openssf-baseline"): + """Write the way run_sieve_audit does: through the resolved store, under the shared key.""" + from darnit.stores.selection import resolve_stores + from darnit.tools.audit import audit_cache_key + + bundle = resolve_stores(None, repo_path=Path(repo)) + try: + write_audit_cache( + str(repo), + results, + summary, + level, + framework, + store=bundle.cache, + cache_key=audit_cache_key(str(repo), None), + ) + finally: + bundle.close_all() + + +def _read_full_audit(repo): + from darnit.tools.audit import read_full_audit_cache + + return read_full_audit_cache(str(repo), "openssf-baseline", level=3) + + +class TestFullAuditCacheRead: + """read_full_audit_cache is what remediation reads (#542).""" + + def test_full_audit_is_a_hit(self, temp_git_repo, sample_results, sample_summary): + _write_full_audit(temp_git_repo, sample_results, sample_summary) + assert _read_full_audit(temp_git_repo) is not None + + def test_level_1_audit_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + _write_full_audit(temp_git_repo, sample_results, sample_summary, level=1) + assert _read_full_audit(temp_git_repo) is None + + def test_other_framework_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + _write_full_audit(temp_git_repo, sample_results, sample_summary, framework="gittuf") + assert _read_full_audit(temp_git_repo) is None + + +class TestFullAuditCacheInvalidation: + """invalidate_full_audit_cache expires what read_full_audit_cache reads (review on #549).""" + + def test_invalidated_cache_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + from darnit.tools.audit import invalidate_full_audit_cache + + _write_full_audit(temp_git_repo, sample_results, sample_summary) + assert _read_full_audit(temp_git_repo) is not None + invalidate_full_audit_cache(str(temp_git_repo), "openssf-baseline") + assert _read_full_audit(temp_git_repo) is None + + def test_configured_store_is_invalidated( + self, temp_git_repo, tmp_path_factory, monkeypatch, sample_results, sample_summary + ): + """Review on #549: with ``[stores.cache]`` set, the configured entry must expire. + + The plain ``invalidate_audit_cache(path)`` expires the default store's + entry and leaves the configured one, so remediation kept reading the + audit from before its own change. + """ + from darnit.config.framework_schema import StoreBlock, StoresConfig + from darnit.core.audit_cache import invalidate_audit_cache + from darnit.stores.selection import resolve_stores + from darnit.tools.audit import audit_cache_key, invalidate_full_audit_cache + + root = tmp_path_factory.mktemp("configured-cache") + stores = StoresConfig(cache=StoreBlock(backend="local-fs", root=str(root))) + monkeypatch.setattr("darnit.tools.audit._load_merged_stores", lambda *args, **kwargs: stores) + + bundle = resolve_stores(stores, repo_path=Path(temp_git_repo)) + try: + write_audit_cache( + str(temp_git_repo), + sample_results, + sample_summary, + 3, + "openssf-baseline", + store=bundle.cache, + cache_key=audit_cache_key(str(temp_git_repo), stores), + ) + finally: + bundle.close_all() + assert list(root.glob("*.json")), "the audit should be cached in the configured store" + assert _read_full_audit(temp_git_repo) is not None + + invalidate_audit_cache(str(temp_git_repo)) + assert _read_full_audit(temp_git_repo) is not None, "the default-store call does not reach it" + + invalidate_full_audit_cache(str(temp_git_repo), "openssf-baseline") + assert _read_full_audit(temp_git_repo) is None diff --git a/tests/darnit/test_cli_audit_cache.py b/tests/darnit/test_cli_audit_cache.py new file mode 100644 index 00000000..74400d1c --- /dev/null +++ b/tests/darnit/test_cli_audit_cache.py @@ -0,0 +1,39 @@ +"""Only a plain `darnit audit` leaves a full-audit cache (#542).""" + +from __future__ import annotations + +import subprocess + +import pytest + +from darnit.cli import main + + +def _repo_with_one_commit(path) -> None: + """A commit is needed: the cache of a repository with no HEAD is always stale.""" + git = ["git", "-C", str(path), "-c", "user.name=test", "-c", "user.email=test@example.com"] + subprocess.run(["git", "init", "-q", str(path)], check=True) + (path / "README.md").write_text("# test\n") + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "-q", "-m", "init"], check=True) + + +@pytest.mark.parametrize( + ("flags", "cached"), + [ + ([], True), + (["-t", "level=1"], False), + (["--include", "OSPS-QA-05.01"], False), + (["--exclude", "OSPS-QA-05.01"], False), + ], +) +def test_only_a_plain_audit_is_cached(tmp_path, flags, cached) -> None: + from darnit.tools.audit import read_full_audit_cache + + _repo_with_one_commit(tmp_path) + + code = main(["audit", "-f", "openssf-baseline", "--no-fail", *flags, str(tmp_path)]) + + assert code == 0 + cache = read_full_audit_cache(str(tmp_path), "openssf-baseline", level=3) + assert (cache is not None) is cached diff --git a/tests/darnit_baseline/remediation/test_outcomes.py b/tests/darnit_baseline/remediation/test_outcomes.py index 706afe63..d264a520 100644 --- a/tests/darnit_baseline/remediation/test_outcomes.py +++ b/tests/darnit_baseline/remediation/test_outcomes.py @@ -240,7 +240,11 @@ def test_run_sieve_audit_without_cache_write(repo: Path) -> None: audit.run_sieve_audit(OWNER, REPO, str(repo), "main", controls=_specs(), framework_name="openssf-baseline", write_cache=False) assert not cache_file.exists() - audit.run_sieve_audit(OWNER, REPO, str(repo), "main", controls=_specs(), framework_name="openssf-baseline") + # cache_filtered=False: a caller that passes its own controls says whether + # they are the full set. Unstated, the audit is cached as filtered (#542). + audit.run_sieve_audit( + OWNER, REPO, str(repo), "main", controls=_specs(), framework_name="openssf-baseline", cache_filtered=False + ) assert cache_file.exists() diff --git a/tests/darnit_baseline/test_audit_cache_scope.py b/tests/darnit_baseline/test_audit_cache_scope.py new file mode 100644 index 00000000..a186c148 --- /dev/null +++ b/tests/darnit_baseline/test_audit_cache_scope.py @@ -0,0 +1,29 @@ +"""A Baseline audit narrowed by a profile is not cached as a full audit (#542).""" + +from __future__ import annotations + +import subprocess + +import pytest + + +def _repo_with_one_commit(path) -> None: + """A commit is needed: the cache of a repository with no HEAD is always stale.""" + git = ["git", "-C", str(path), "-c", "user.name=test", "-c", "user.email=test@example.com"] + subprocess.run(["git", "init", "-q", str(path)], check=True) + (path / "README.md").write_text("# test\n") + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "-q", "-m", "init"], check=True) + + +@pytest.mark.parametrize(("profile", "cached"), [("level1_quick", False), (None, True)]) +def test_only_an_unfiltered_audit_is_cached(tmp_path, profile, cached) -> None: + from darnit.tools.audit import read_full_audit_cache + from darnit_baseline.tools import audit_openssf_baseline + + _repo_with_one_commit(tmp_path) + + audit_openssf_baseline(local_path=str(tmp_path), profile=profile, auto_init_config=False) + + cache = read_full_audit_cache(str(tmp_path), "openssf-baseline", level=3) + assert (cache is not None) is cached