From b9d9ec693b1e2622321e44eba3e1158160715b92 Mon Sep 17 00:00:00 2001 From: Marc-cn <130138935+Marc-cn@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:32:50 -0400 Subject: [PATCH 1/3] fix(cache): record audit scope and reject mismatched cache reads (#542) A filtered audit left a cache that remediation read as a full audit. On main, `darnit audit --tags level=1` writes 26 results and remediate_audit_findings then acts on those as if all 66 controls had run. The cache envelope now records the audit's scope: framework, level, tag filter, and whether any filter narrowed the control set. CACHE_VERSION is 2 and must match exactly, so older envelopes, which cannot describe their own scope, are ignored and rebuilt. A filtered audit is written under its own key, so it never replaces the full-audit entry. run_sieve_audit records an audit as filtered when it has tags, when the caller says so, or when the caller passed its own `controls` without saying. That last case fails closed: the function cannot tell a narrowed list from the full set. Callers with the full set say cache_filtered=False (the generic MCP audit tool); the CLI and the Baseline tool compute it from --tags/--include/--exclude and from the profile. Remediation reads through read_full_audit_cache, which resolves the operator configuration and the store the way run_sieve_audit does, derives the key with the same helper (audit_cache_key), and accepts only an unfiltered level 3 audit of its framework. Rebuilt on current main rather than rebased: main had rewritten the three conflicting files since the original commit. Drafted with Claude, edited by me. Signed-off-by: Marc-cn <130138935+Marc-cn@users.noreply.github.com> Assisted-by: Claude:claude-fable-5-1 --- CHANGELOG.md | 7 + docs/architecture/framework-design.md | 18 +- .../remediation/orchestrator.py | 9 +- .../src/darnit_baseline/tools.py | 3 + packages/darnit/src/darnit/cli.py | 5 + .../darnit/src/darnit/core/audit_cache.py | 55 ++++++- .../src/darnit/server/tools/builtin_audit.py | 3 + packages/darnit/src/darnit/tools/audit.py | 84 +++++++++- tests/darnit/core/test_audit_cache.py | 154 ++++++++++++++++++ .../remediation/test_outcomes.py | 6 +- .../darnit_baseline/test_audit_cache_scope.py | 28 ++++ 11 files changed, 362 insertions(+), 10 deletions(-) create mode 100644 tests/darnit_baseline/test_audit_cache_scope.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 749c1189..6523c702 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -588,6 +588,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 grepping the text report, so a reworded message cannot hide a broken dependency, and it refuses a `dist/` holding more than one wheel of a package (#575). +- A filtered audit is no longer reused as a full one (#542). The audit + cache records the audit's scope: framework, level, tag filter, and + whether any filter narrowed the control set. A filtered audit is cached + under its own key, and remediation reads only an unfiltered level 3 + audit of its framework, through the store the audit wrote to. Before, + `darnit audit --tags level=1` left a cache that remediation treated as + a full audit. Cache files from earlier versions are ignored and rebuilt. - Installs without the development dependencies work again (`uv tool install` from a checkout, and any install from a built wheel). Two things broke them: `tree-sitter-language-pack` 1.6.3, the newest release the old diff --git a/docs/architecture/framework-design.md b/docs/architecture/framework-design.md index e09db85f..811d0df6 100644 --- a/docs/architecture/framework-design.md +++ b/docs/architecture/framework-design.md @@ -1870,8 +1870,24 @@ Cache files are stored in the system temp directory (`$TMPDIR/darnit/ - **AND** controls with `status == "WARN"` SHALL NOT be remediated (WARN means automated verification was inconclusive, not that the control is non-compliant) - **AND** controls with `status == "PASS"` SHALL NOT be remediated +#### Requirement: The cache envelope records the audit's scope +- **WHEN** `write_audit_cache()` is called +- **THEN** the envelope SHALL record `framework`, `level`, `tags` (the tag filter, empty when none) and `filtered` (true when any filter narrowed the control set, including `--include` and `--exclude`) +- **AND** a filtered audit SHALL be written under a distinct cache key, so it does not replace an unfiltered entry +- **AND** `run_sieve_audit()` SHALL record an audit as filtered when it has tags, or when its caller passed `controls` without stating `cache_filtered`; a caller that passes the full control set SHALL state `cache_filtered=False` +- **AND** the cache key SHALL be derived in one place (`audit_cache_key()`), used by the writer and by every reader + +#### Requirement: A cache read may constrain the scope it accepts +- **WHEN** `read_audit_cache()` is called with any of `expected_framework`, `expected_level`, `expected_tags` or `expected_filtered` +- **THEN** a mismatch on any supplied expectation SHALL be treated as a cache miss +- **AND** a caller supplying none of them SHALL be unaffected +- **AND** an envelope whose `version` differs from `CACHE_VERSION` SHALL be a miss, since an older envelope cannot express its own scope + #### Requirement: Remediation consumes cached audit results -- **WHEN** `remediate_audit_findings()` is called and `read_audit_cache()` returns valid cached results +- **WHEN** `remediate_audit_findings()` is called +- **THEN** it SHALL read through the store selected by `[stores.cache]`, the same store the audit writes to +- **AND** it SHALL require an unfiltered level-3 `openssf-baseline` cache, matching the audit it would otherwise run +- **WHEN** `read_audit_cache()` returns valid cached results - **THEN** it SHALL extract failed control IDs from the cached results (entries with `status == "FAIL"`) - **AND** it SHALL NOT run a redundant audit - **WHEN** `read_audit_cache()` returns `None` (cache miss) diff --git a/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py b/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py index 838d0f38..2281d85d 100644 --- a/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py +++ b/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py @@ -1117,8 +1117,13 @@ def run_remediation( error: str | None = None try: - from darnit.core.audit_cache import read_audit_cache - cache = read_audit_cache(local_path) + # Issue #542: remediation acts on a full level-3 audit of this + # framework, which is what it runs on a miss. A level-1, filtered or + # other-framework cache is a subset, so it must miss. The read goes + # through the store and key the audit writes to. + from darnit.tools.audit import read_full_audit_cache + + cache = read_full_audit_cache(local_path, "openssf-baseline", level=3) except Exception as exc: logger.warning(f"Audit cache read failed: {exc}") cache = None diff --git a/packages/darnit-baseline/src/darnit_baseline/tools.py b/packages/darnit-baseline/src/darnit_baseline/tools.py index 4f68a031..f497fbad 100644 --- a/packages/darnit-baseline/src/darnit_baseline/tools.py +++ b/packages/darnit-baseline/src/darnit_baseline/tools.py @@ -226,6 +226,9 @@ def audit_openssf_baseline( level=level, controls=controls, tags=tags_list, + # Issue #542: a profile narrows `controls` above. Tags are + # recorded by run_sieve_audit itself. + cache_filtered=bool(profile), stop_on_llm=True, framework_name="openssf-baseline", operator_config=operator_config, diff --git a/packages/darnit/src/darnit/cli.py b/packages/darnit/src/darnit/cli.py index d056c9d4..bd0ee494 100644 --- a/packages/darnit/src/darnit/cli.py +++ b/packages/darnit/src/darnit/cli.py @@ -334,6 +334,11 @@ def cmd_audit(args: argparse.Namespace) -> int: default_branch=default_branch, level=3, controls=controls, + # Issue #542: the CLI narrows `controls` itself, so it says whether + # it did. Without this a `--tags domain=VM` run would be cached as + # if it were a full audit. + cache_tags=",".join(sorted(args.tags)) if args.tags else "", + cache_filtered=bool(args.tags or args.include or args.exclude), stop_on_llm=True, # Issue #427: the framework name has to reach the audit driver, not # just the control loader above. Without it the driver cannot diff --git a/packages/darnit/src/darnit/core/audit_cache.py b/packages/darnit/src/darnit/core/audit_cache.py index 9e587025..e14432b5 100644 --- a/packages/darnit/src/darnit/core/audit_cache.py +++ b/packages/darnit/src/darnit/core/audit_cache.py @@ -55,7 +55,7 @@ logger = get_logger("core.audit_cache") CACHE_FILENAME = "audit-cache.json" -CACHE_VERSION = 1 +CACHE_VERSION = 2 # Invalidation via write-expired-envelope (feature 035 clarify Q3). # The AuditCacheStore Protocol has no delete(key) method; instead, @@ -68,6 +68,8 @@ "commit_dirty": False, "level": 0, "framework": "", + "tags": "", + "filtered": False, "results": [], "summary": {}, } @@ -179,6 +181,8 @@ def write_audit_cache( level: int, framework: str, *, + tags: str = "", + filtered: bool = False, store: AuditCacheStore | None = None, cache_key: str | None = None, ) -> None: @@ -214,6 +218,13 @@ def write_audit_cache( "commit_dirty": _is_working_tree_dirty(local_path), "level": level, "framework": framework, + # Issue #542: the scope the audit ran at. A later read that expects a + # different framework, level or tag filter must miss, not silently + # reuse a subset as if it were a full audit. + "tags": tags, + # True when ANY filter narrowed the control set: tags, --include, + # --exclude, or a profile. `tags` alone cannot express the others. + "filtered": filtered, "results": results, "summary": summary, } @@ -232,6 +243,10 @@ def read_audit_cache( local_path: str, ttl_seconds: int = 3600, *, + expected_framework: str | None = None, + expected_level: int | None = None, + expected_tags: str | None = None, + expected_filtered: bool | None = None, store: AuditCacheStore | None = None, cache_key: str | None = None, ) -> dict[str, Any] | None: @@ -266,10 +281,46 @@ def read_audit_cache( # Version check version = data.get("version") - if not isinstance(version, int) or version > CACHE_VERSION: + # Issue #542 review: an exact match, not an upper bound. A v1 envelope + # has no `tags`/`filtered` keys, so reading it would present a possibly + # filtered audit as unfiltered. Older caches miss and are re-audited. + if not isinstance(version, int) or version != CACHE_VERSION: logger.debug("Unknown audit cache version: %s", version) return None + # Scope: the cached audit must cover what the caller is about to use it + # for (issue #542). A level-1, tag-filtered or other-framework run leaves a + # cache that is a subset of a full Baseline audit; reusing it would make a + # consumer act on results that were never produced. + if expected_framework is not None and data.get("framework") != expected_framework: + logger.debug( + "Audit cache scope mismatch: framework %r != expected %r", + data.get("framework"), + expected_framework, + ) + return None + if expected_level is not None and data.get("level") != expected_level: + logger.debug( + "Audit cache scope mismatch: level %r != expected %r", + data.get("level"), + expected_level, + ) + return None + if expected_filtered is not None and bool(data.get("filtered")) != expected_filtered: + logger.debug( + "Audit cache scope mismatch: filtered %r != expected %r", + data.get("filtered"), + expected_filtered, + ) + return None + if expected_tags is not None and (data.get("tags") or "") != expected_tags: + logger.debug( + "Audit cache scope mismatch: tags %r != expected %r", + data.get("tags"), + expected_tags, + ) + return None + # Staleness: TTL expiry timestamp_str = data.get("timestamp") if isinstance(timestamp_str, str): diff --git a/packages/darnit/src/darnit/server/tools/builtin_audit.py b/packages/darnit/src/darnit/server/tools/builtin_audit.py index 833af0a3..8212ebd2 100644 --- a/packages/darnit/src/darnit/server/tools/builtin_audit.py +++ b/packages/darnit/src/darnit/server/tools/builtin_audit.py @@ -133,6 +133,9 @@ async def builtin_audit( level=level, controls=all_controls, tags=tags_list, + # Issue #542: all_controls is the framework's full set. Tags are + # recorded by run_sieve_audit itself. + cache_filtered=False, stop_on_llm=True, framework_name=_framework_name, operator_config=operator_config, diff --git a/packages/darnit/src/darnit/tools/audit.py b/packages/darnit/src/darnit/tools/audit.py index ed76c33b..63025648 100644 --- a/packages/darnit/src/darnit/tools/audit.py +++ b/packages/darnit/src/darnit/tools/audit.py @@ -167,6 +167,78 @@ def _load_merged_stores(framework_name: str | None, operator: "OperatorConfig | return getattr(effective, "stores", None) +def audit_cache_key(local_path: str, stores_config: Any, *, filtered: bool = False) -> str: + """Return the key an audit of ``local_path`` is cached under. + + The one derivation for the writer (:func:`run_sieve_audit`) and for every + reader. The default store's root already encodes the repository, so the + key is fixed; a configured store may be shared across repositories, so the + key encodes the path. A filtered audit gets its own key, so it never + replaces the full-audit entry (issue #542). + """ + if stores_config is None or stores_config.cache is None: + key = "audit-cache" + else: + key = hashlib.sha256(str(Path(local_path).resolve()).encode()).hexdigest()[:16] + return f"{key}-filtered" if filtered else key + + +def _cache_scope( + tags: list[str] | None, + controls: list | None, + cache_tags: str | None, + cache_filtered: bool | None, +) -> tuple[str, bool]: + """Return the (tags, filtered) pair recorded in the audit cache (issue #542). + + Tags always mean filtered. Otherwise the caller's word decides. A caller + that passed its own ``controls`` and did not say is treated as filtered: + this function cannot tell a narrowed list from the full set, and a subset + read back as a full audit makes remediation act on results that were never + produced. + """ + if cache_tags is None: + cache_tags = ",".join(sorted(tags)) if tags else "" + if cache_filtered is None: + cache_filtered = controls is not None + return cache_tags, bool(cache_tags) or cache_filtered + + +def read_full_audit_cache( + local_path: str, + framework_name: str, + level: int, + operator_config: "LoadedOperatorConfig | None" = None, +) -> dict[str, Any] | None: + """Return the cached unfiltered audit of ``framework_name`` at ``level``, or None. + + Reads through the store and key :func:`run_sieve_audit` writes to, so an + operator's ``[stores.cache]`` is honoured on both sides. A cache from + another framework, another level or a filtered run is a miss. + """ + from darnit.core.audit_cache import read_audit_cache + from darnit.stores.selection import resolve_stores + + if operator_config is None: + from darnit.config.operator.loader import resolve_operator_config + + operator_config = resolve_operator_config(local_path) + stores_config = _load_merged_stores(framework_name, operator_config.config) + stores_bundle = resolve_stores(stores_config, repo_path=Path(local_path)) + try: + return read_audit_cache( + local_path, + expected_framework=framework_name, + expected_level=level, + expected_tags="", + expected_filtered=False, + store=stores_bundle.cache, + cache_key=audit_cache_key(local_path, stores_config, filtered=False), + ) + finally: + stores_bundle.close_all() + + def _load_merged_mcp_servers(framework_name: str | None, operator: "OperatorConfig | None" = None) -> dict[str, Any]: """Return the merged ``mcp_servers`` allowlist for this audit run. @@ -731,6 +803,8 @@ def run_sieve_audit( *, controls: list | None = None, tags: list[str] | None = None, + cache_tags: str | None = None, + cache_filtered: bool | None = None, evaluate_claims: bool = True, stop_on_llm: bool = True, framework_name: str | None = None, @@ -1014,10 +1088,10 @@ def run_sieve_audit( # already encodes repo identity (uses fixed "audit-cache"); configured # store's root is operator-picked and may be shared across repos, so # the key must encode repo identity. - if stores_config is None or stores_config.cache is None: - cache_key = "audit-cache" - else: - cache_key = hashlib.sha256(str(Path(local_path).resolve()).encode()).hexdigest()[:16] + # Issue #542: record the scope this audit ran at, and keep a filtered run + # out of the full-audit entry. + cache_tags, cache_filtered = _cache_scope(tags, controls, cache_tags, cache_filtered) + cache_key = audit_cache_key(local_path, stores_config, filtered=cache_filtered) from darnit.core.audit_cache import write_audit_cache @@ -1033,6 +1107,8 @@ def run_sieve_audit( summary, level, resolved_fw or "", + tags=cache_tags, + filtered=cache_filtered, store=stores_bundle.cache, cache_key=cache_key, ) diff --git a/tests/darnit/core/test_audit_cache.py b/tests/darnit/core/test_audit_cache.py index d93ae306..dba6368c 100644 --- a/tests/darnit/core/test_audit_cache.py +++ b/tests/darnit/core/test_audit_cache.py @@ -135,6 +135,8 @@ def test_envelope_structure(self, temp_git_repo: Path, sample_results, sample_su "commit_dirty", "level", "framework", + "tags", + "filtered", "results", "summary", } @@ -475,3 +477,155 @@ def worker(thread_id): final_data = read_audit_cache(repo_path_str) assert final_data is not None assert final_data["version"] == CACHE_VERSION + + +class TestScopeMismatch: + """Issue #542: a cache from a narrower audit must not satisfy a wider read.""" + + def test_framework_mismatch_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 3, "openssf-baseline") + assert read_audit_cache(str(temp_git_repo), expected_framework="openssf-baseline") is not None + assert read_audit_cache(str(temp_git_repo), expected_framework="amber") is None + + def test_level_mismatch_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 1, "openssf-baseline") + assert read_audit_cache(str(temp_git_repo), expected_level=1) is not None + assert read_audit_cache(str(temp_git_repo), expected_level=3) is None + + def test_tag_filtered_cache_does_not_satisfy_an_unfiltered_read( + self, temp_git_repo, sample_results, sample_summary + ): + write_audit_cache( + str(temp_git_repo), + sample_results, + sample_summary, + 3, + "openssf-baseline", + tags="domain=VM", + ) + assert read_audit_cache(str(temp_git_repo), expected_tags="domain=VM") is not None + assert read_audit_cache(str(temp_git_repo), expected_tags="") is None + + def test_unscoped_read_still_hits(self, temp_git_repo, sample_results, sample_summary): + """Existing callers that pass no expectations are unaffected.""" + write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 1, "amber", tags="domain=VM") + assert read_audit_cache(str(temp_git_repo)) is not None + + +class TestCacheVersionAndFilteredFlag: + """Review on #549: v1 envelopes must miss, and `filtered` covers the + filters a tag string cannot express (--include/--exclude/profile).""" + + def test_old_version_envelope_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 3, "openssf-baseline") + cache_path = _get_cache_dir(str(temp_git_repo)) / CACHE_FILENAME + data = json.loads(cache_path.read_text()) + data["version"] = 1 + data.pop("tags", None) + data.pop("filtered", None) + cache_path.write_text(json.dumps(data)) + assert read_audit_cache(str(temp_git_repo)) is None + + def test_filtered_cache_does_not_satisfy_an_unfiltered_read(self, temp_git_repo, sample_results, sample_summary): + write_audit_cache( + str(temp_git_repo), + sample_results, + sample_summary, + 3, + "openssf-baseline", + filtered=True, + ) + assert read_audit_cache(str(temp_git_repo), expected_filtered=True) is not None + assert read_audit_cache(str(temp_git_repo), expected_filtered=False) is None + + def test_filtered_is_independent_of_tags(self, temp_git_repo, sample_results, sample_summary): + """--include narrows the set without producing a tag string.""" + write_audit_cache( + str(temp_git_repo), + sample_results, + sample_summary, + 3, + "openssf-baseline", + tags="", + filtered=True, + ) + assert read_audit_cache(str(temp_git_repo), expected_tags="") is not None + assert read_audit_cache(str(temp_git_repo), expected_filtered=False) is None + + +class TestCacheScopeDecision: + """What run_sieve_audit records as an audit's scope (#542, review on #549).""" + + @pytest.mark.parametrize( + ("tags", "controls", "cache_tags", "cache_filtered", "expected"), + [ + (None, None, None, None, ("", False)), # darnit loads the controls: a full audit + (["domain=VM"], None, None, None, ("domain=VM", True)), + (None, ["c"], None, None, ("", True)), # caller's own list, not declared: fail closed + (None, ["c"], None, False, ("", False)), # caller says it is the full set + (None, ["c"], None, True, ("", True)), # caller says it narrowed, e.g. a profile + (["level=1"], ["c"], None, False, ("level=1", True)), # tags win over "not filtered" + (None, ["c"], "domain=VM", False, ("domain=VM", True)), + ], + ) + def test_scope(self, tags, controls, cache_tags, cache_filtered, expected): + from darnit.tools.audit import _cache_scope + + assert _cache_scope(tags, controls, cache_tags, cache_filtered) == expected + + def test_filtered_audits_get_their_own_key(self): + from darnit.tools.audit import audit_cache_key + + assert audit_cache_key("/repo", None, filtered=False) == "audit-cache" + assert audit_cache_key("/repo", None, filtered=True) == "audit-cache-filtered" + + +class TestFullAuditCacheRead: + """read_full_audit_cache is what remediation reads (#542).""" + + @staticmethod + def _write(repo, results, summary, *, level=3, framework="openssf-baseline", filtered=False, tags=""): + # Write the way run_sieve_audit does: through the resolved store, + # under the shared key. + from darnit.stores.selection import resolve_stores + from darnit.tools.audit import audit_cache_key + + bundle = resolve_stores(None, repo_path=Path(repo)) + try: + write_audit_cache( + str(repo), results, summary, level, framework, + tags=tags, filtered=filtered, + store=bundle.cache, + cache_key=audit_cache_key(str(repo), None, filtered=filtered), + ) + finally: + bundle.close_all() + + @staticmethod + def _read(repo): + from darnit.tools.audit import read_full_audit_cache + + return read_full_audit_cache(str(repo), "openssf-baseline", level=3) + + def test_full_audit_is_a_hit(self, temp_git_repo, sample_results, sample_summary): + self._write(temp_git_repo, sample_results, sample_summary) + assert self._read(temp_git_repo) is not None + + def test_filtered_audit_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + self._write(temp_git_repo, sample_results, sample_summary, filtered=True) + assert self._read(temp_git_repo) is None + + def test_filtered_audit_does_not_replace_the_full_one(self, temp_git_repo, sample_results, sample_summary): + self._write(temp_git_repo, sample_results, sample_summary) + self._write(temp_git_repo, [], {}, filtered=True, tags="domain=VM") + cache = self._read(temp_git_repo) + assert cache is not None + assert cache["results"] == sample_results + + def test_level_1_audit_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + self._write(temp_git_repo, sample_results, sample_summary, level=1) + assert self._read(temp_git_repo) is None + + def test_other_framework_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + self._write(temp_git_repo, sample_results, sample_summary, framework="gittuf") + assert self._read(temp_git_repo) is None diff --git a/tests/darnit_baseline/remediation/test_outcomes.py b/tests/darnit_baseline/remediation/test_outcomes.py index 706afe63..d264a520 100644 --- a/tests/darnit_baseline/remediation/test_outcomes.py +++ b/tests/darnit_baseline/remediation/test_outcomes.py @@ -240,7 +240,11 @@ def test_run_sieve_audit_without_cache_write(repo: Path) -> None: audit.run_sieve_audit(OWNER, REPO, str(repo), "main", controls=_specs(), framework_name="openssf-baseline", write_cache=False) assert not cache_file.exists() - audit.run_sieve_audit(OWNER, REPO, str(repo), "main", controls=_specs(), framework_name="openssf-baseline") + # cache_filtered=False: a caller that passes its own controls says whether + # they are the full set. Unstated, the audit is cached as filtered (#542). + audit.run_sieve_audit( + OWNER, REPO, str(repo), "main", controls=_specs(), framework_name="openssf-baseline", cache_filtered=False + ) assert cache_file.exists() diff --git a/tests/darnit_baseline/test_audit_cache_scope.py b/tests/darnit_baseline/test_audit_cache_scope.py new file mode 100644 index 00000000..985505c0 --- /dev/null +++ b/tests/darnit_baseline/test_audit_cache_scope.py @@ -0,0 +1,28 @@ +"""The Baseline audit tool tells the cache when a profile narrowed the audit (#542).""" + +from __future__ import annotations + +import subprocess + +import pytest + + +@pytest.mark.parametrize(("profile", "expected"), [("level1_quick", True), (None, False)]) +def test_profile_audit_is_recorded_as_filtered(tmp_path, monkeypatch, profile, expected) -> None: + captured: dict = {} + + def recording_audit(**kwargs): + captured.update(kwargs) + return [], {} + + monkeypatch.setattr("darnit.tools.audit.run_sieve_audit", recording_audit) + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + + from darnit_baseline.tools import audit_openssf_baseline + + try: + audit_openssf_baseline(local_path=str(tmp_path), profile=profile, auto_init_config=False) + except Exception: # noqa: BLE001 - only the arguments handed to the audit matter here + pass + + assert captured["cache_filtered"] is expected From aff2c503738a1636cd926c0ae1dd3b3c78790cd0 Mon Sep 17 00:00:00 2001 From: Marc-cn <130138935+Marc-cn@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:29:59 -0400 Subject: [PATCH 2/3] fix(cache): do not cache filtered audits, invalidate through the configured store Addresses the review of Oct 12 on this PR. - A filtered audit is not written to the cache at all. That replaces the separate "-filtered" key and the tags/filtered envelope fields, which nothing read. The read checks framework and level only. - invalidate_full_audit_cache() resolves the same store and key as read_full_audit_cache(). Remediation used the default store, so with a configured [stores.cache] the old audit stayed in place. - The CLI derives cache_filtered from the control count: fewer controls than were loaded means filtered. That covers --profile (#576) and any future flag without listing them. - The profile test runs the real audit_openssf_baseline() and checks that a profile run leaves no full-audit cache. - Spec 10.4 updated for the invalidation and the envelope. Assisted-by: Claude:claude-fable-5-1 Signed-off-by: Marc-cn <130138935+Marc-cn@users.noreply.github.com> --- CHANGELOG.md | 9 + docs/architecture/framework-design.md | 16 +- .../remediation/orchestrator.py | 4 +- .../src/darnit_baseline/tools.py | 4 +- packages/darnit/src/darnit/cli.py | 9 +- .../darnit/src/darnit/core/audit_cache.py | 34 +-- packages/darnit/src/darnit/tools/audit.py | 109 ++++++---- tests/darnit/core/test_audit_cache.py | 199 +++++++++--------- .../darnit_baseline/test_audit_cache_scope.py | 31 +-- 9 files changed, 213 insertions(+), 202 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6523c702..7d58f467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -595,6 +595,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 audit of its framework, through the store the audit wrote to. Before, `darnit audit --tags level=1` left a cache that remediation treated as a full audit. Cache files from earlier versions are ignored and rebuilt. +- A filtered audit is no longer reused as a full one (#542). Only an + unfiltered audit is cached: a run narrowed by `--tags`, `--include`, + `--exclude` or a profile does not write the audit cache. Remediation + reads only a level 3 audit of its framework, through the store the + audit wrote to, and expires it through that same store after it changes + something. Before, `darnit audit --tags level=1` left a cache that + remediation treated as a full audit, and with a configured + `[stores.cache]` a remediation left the old audit in place. Cache files + from earlier versions are ignored and rebuilt. - Installs without the development dependencies work again (`uv tool install` from a checkout, and any install from a built wheel). Two things broke them: `tree-sitter-language-pack` 1.6.3, the newest release the old diff --git a/docs/architecture/framework-design.md b/docs/architecture/framework-design.md index 811d0df6..e7d8f8ff 100644 --- a/docs/architecture/framework-design.md +++ b/docs/architecture/framework-design.md @@ -1872,16 +1872,16 @@ Cache files are stored in the system temp directory (`$TMPDIR/darnit/ #### Requirement: The cache envelope records the audit's scope - **WHEN** `write_audit_cache()` is called -- **THEN** the envelope SHALL record `framework`, `level`, `tags` (the tag filter, empty when none) and `filtered` (true when any filter narrowed the control set, including `--include` and `--exclude`) -- **AND** a filtered audit SHALL be written under a distinct cache key, so it does not replace an unfiltered entry -- **AND** `run_sieve_audit()` SHALL record an audit as filtered when it has tags, or when its caller passed `controls` without stating `cache_filtered`; a caller that passes the full control set SHALL state `cache_filtered=False` -- **AND** the cache key SHALL be derived in one place (`audit_cache_key()`), used by the writer and by every reader +- **THEN** the envelope SHALL record `framework` and `level` +- **AND** `run_sieve_audit()` SHALL NOT write the cache for a filtered audit, so a subset of the controls is never stored where a full audit is expected +- **AND** `run_sieve_audit()` SHALL treat an audit as filtered when it has tags, or when its caller passed `controls` without stating `cache_filtered`; a caller that passes the full control set SHALL state `cache_filtered=False` +- **AND** the cache key SHALL be derived in one place (`audit_cache_key()`), used by the writer, the reader and the invalidation #### Requirement: A cache read may constrain the scope it accepts -- **WHEN** `read_audit_cache()` is called with any of `expected_framework`, `expected_level`, `expected_tags` or `expected_filtered` +- **WHEN** `read_audit_cache()` is called with `expected_framework` or `expected_level` - **THEN** a mismatch on any supplied expectation SHALL be treated as a cache miss - **AND** a caller supplying none of them SHALL be unaffected -- **AND** an envelope whose `version` differs from `CACHE_VERSION` SHALL be a miss, since an older envelope cannot express its own scope +- **AND** an envelope whose `version` differs from `CACHE_VERSION` SHALL be a miss, since an older envelope may hold a filtered audit #### Requirement: Remediation consumes cached audit results - **WHEN** `remediate_audit_findings()` is called @@ -1896,9 +1896,9 @@ Cache files are stored in the system temp directory (`$TMPDIR/darnit/ #### Requirement: Post-remediation cache invalidation - **WHEN** `remediate_audit_findings()` completes with `dry_run=False` and at least one outcome changed something (section 15.4) -- **THEN** it SHALL call `invalidate_audit_cache(local_path)` +- **THEN** it SHALL expire the cached full audit through the same store and key the cache read uses (`invalidate_full_audit_cache(local_path, framework_name)`), so that a configured `[stores.cache]` is invalidated and not only the default store - **WHEN** `remediate_audit_findings()` completes with `dry_run=True` -- **THEN** it SHALL NOT call `invalidate_audit_cache()` +- **THEN** it SHALL NOT invalidate the cache ### 10.5 Framework Contains No Implementation-Specific Code diff --git a/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py b/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py index 2281d85d..b868077f 100644 --- a/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py +++ b/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py @@ -1284,9 +1284,9 @@ def run_remediation( outcomes = _outcomes(results, lambda ids: _recheck(ids, local_path, owner, repo, target)) if any(_written(r) or _applied_change_sets(r) for r in results): try: - from darnit.core import audit_cache + from darnit.tools.audit import invalidate_full_audit_cache - audit_cache.invalidate_audit_cache(local_path) + invalidate_full_audit_cache(local_path, "openssf-baseline") except Exception as exc: logger.warning(f"Failed to invalidate audit cache: {exc}") diff --git a/packages/darnit-baseline/src/darnit_baseline/tools.py b/packages/darnit-baseline/src/darnit_baseline/tools.py index f497fbad..8cd42360 100644 --- a/packages/darnit-baseline/src/darnit_baseline/tools.py +++ b/packages/darnit-baseline/src/darnit_baseline/tools.py @@ -507,9 +507,9 @@ def create_security_policy( written = [c.path for c in outcome.file_changes if c.changes] if written: try: - from darnit.core import audit_cache + from darnit.tools.audit import invalidate_full_audit_cache - audit_cache.invalidate_audit_cache(str(repo_path)) + invalidate_full_audit_cache(str(repo_path), "openssf-baseline") except Exception as exc: # noqa: BLE001 from darnit.core.logging import get_logger diff --git a/packages/darnit/src/darnit/cli.py b/packages/darnit/src/darnit/cli.py index bd0ee494..b9732d16 100644 --- a/packages/darnit/src/darnit/cli.py +++ b/packages/darnit/src/darnit/cli.py @@ -312,6 +312,8 @@ def cmd_audit(args: argparse.Namespace) -> int: include_ids = set(args.include.split(",")) if args.include else None exclude_ids = set(args.exclude.split(",")) if args.exclude else set() + loaded_count = len(controls) + # Apply filters controls = filter_controls(controls, filters, include_ids, exclude_ids) @@ -335,10 +337,9 @@ def cmd_audit(args: argparse.Namespace) -> int: level=3, controls=controls, # Issue #542: the CLI narrows `controls` itself, so it says whether - # it did. Without this a `--tags domain=VM` run would be cached as - # if it were a full audit. - cache_tags=",".join(sorted(args.tags)) if args.tags else "", - cache_filtered=bool(args.tags or args.include or args.exclude), + # it did. Comparing the counts covers every filter, present and + # future, without listing the flags. A filtered audit is not cached. + cache_filtered=len(controls) < loaded_count, stop_on_llm=True, # Issue #427: the framework name has to reach the audit driver, not # just the control loader above. Without it the driver cannot diff --git a/packages/darnit/src/darnit/core/audit_cache.py b/packages/darnit/src/darnit/core/audit_cache.py index e14432b5..3e7891d8 100644 --- a/packages/darnit/src/darnit/core/audit_cache.py +++ b/packages/darnit/src/darnit/core/audit_cache.py @@ -68,8 +68,6 @@ "commit_dirty": False, "level": 0, "framework": "", - "tags": "", - "filtered": False, "results": [], "summary": {}, } @@ -181,8 +179,6 @@ def write_audit_cache( level: int, framework: str, *, - tags: str = "", - filtered: bool = False, store: AuditCacheStore | None = None, cache_key: str | None = None, ) -> None: @@ -218,13 +214,6 @@ def write_audit_cache( "commit_dirty": _is_working_tree_dirty(local_path), "level": level, "framework": framework, - # Issue #542: the scope the audit ran at. A later read that expects a - # different framework, level or tag filter must miss, not silently - # reuse a subset as if it were a full audit. - "tags": tags, - # True when ANY filter narrowed the control set: tags, --include, - # --exclude, or a profile. `tags` alone cannot express the others. - "filtered": filtered, "results": results, "summary": summary, } @@ -245,8 +234,6 @@ def read_audit_cache( *, expected_framework: str | None = None, expected_level: int | None = None, - expected_tags: str | None = None, - expected_filtered: bool | None = None, store: AuditCacheStore | None = None, cache_key: str | None = None, ) -> dict[str, Any] | None: @@ -289,9 +276,10 @@ def read_audit_cache( return None # Scope: the cached audit must cover what the caller is about to use it - # for (issue #542). A level-1, tag-filtered or other-framework run leaves a - # cache that is a subset of a full Baseline audit; reusing it would make a - # consumer act on results that were never produced. + # for (issue #542). A level-1 or other-framework run leaves a cache that + # is not the audit the caller asked for; reusing it would make a consumer + # act on results that were never produced. Filtered audits are never + # written, see run_sieve_audit. if expected_framework is not None and data.get("framework") != expected_framework: logger.debug( "Audit cache scope mismatch: framework %r != expected %r", @@ -306,20 +294,6 @@ def read_audit_cache( expected_level, ) return None - if expected_filtered is not None and bool(data.get("filtered")) != expected_filtered: - logger.debug( - "Audit cache scope mismatch: filtered %r != expected %r", - data.get("filtered"), - expected_filtered, - ) - return None - if expected_tags is not None and (data.get("tags") or "") != expected_tags: - logger.debug( - "Audit cache scope mismatch: tags %r != expected %r", - data.get("tags"), - expected_tags, - ) - return None # Staleness: TTL expiry timestamp_str = data.get("timestamp") diff --git a/packages/darnit/src/darnit/tools/audit.py b/packages/darnit/src/darnit/tools/audit.py index 63025648..74aa0962 100644 --- a/packages/darnit/src/darnit/tools/audit.py +++ b/packages/darnit/src/darnit/tools/audit.py @@ -167,29 +167,22 @@ def _load_merged_stores(framework_name: str | None, operator: "OperatorConfig | return getattr(effective, "stores", None) -def audit_cache_key(local_path: str, stores_config: Any, *, filtered: bool = False) -> str: - """Return the key an audit of ``local_path`` is cached under. +def audit_cache_key(local_path: str, stores_config: Any) -> str: + """Return the key the full audit of ``local_path`` is cached under. - The one derivation for the writer (:func:`run_sieve_audit`) and for every - reader. The default store's root already encodes the repository, so the + The one derivation for the writer (:func:`run_sieve_audit`), the reader + and the invalidation. The default store's root already encodes the repository, so the key is fixed; a configured store may be shared across repositories, so the - key encodes the path. A filtered audit gets its own key, so it never - replaces the full-audit entry (issue #542). + key encodes the path. Only unfiltered audits are cached (issue #542), so + there is one entry per repository. """ if stores_config is None or stores_config.cache is None: - key = "audit-cache" - else: - key = hashlib.sha256(str(Path(local_path).resolve()).encode()).hexdigest()[:16] - return f"{key}-filtered" if filtered else key + return "audit-cache" + return hashlib.sha256(str(Path(local_path).resolve()).encode()).hexdigest()[:16] -def _cache_scope( - tags: list[str] | None, - controls: list | None, - cache_tags: str | None, - cache_filtered: bool | None, -) -> tuple[str, bool]: - """Return the (tags, filtered) pair recorded in the audit cache (issue #542). +def _is_filtered(tags: list[str] | None, controls: list | None, cache_filtered: bool | None) -> bool: + """Return whether this audit covers less than the full control set (issue #542). Tags always mean filtered. Otherwise the caller's word decides. A caller that passed its own ``controls`` and did not say is treated as filtered: @@ -197,26 +190,24 @@ def _cache_scope( read back as a full audit makes remediation act on results that were never produced. """ - if cache_tags is None: - cache_tags = ",".join(sorted(tags)) if tags else "" + if tags: + return True if cache_filtered is None: - cache_filtered = controls is not None - return cache_tags, bool(cache_tags) or cache_filtered + return controls is not None + return cache_filtered -def read_full_audit_cache( +def _full_audit_cache_location( local_path: str, framework_name: str, - level: int, - operator_config: "LoadedOperatorConfig | None" = None, -) -> dict[str, Any] | None: - """Return the cached unfiltered audit of ``framework_name`` at ``level``, or None. + operator_config: "LoadedOperatorConfig | None", +) -> tuple[Any, str]: + """Return the (stores bundle, key) the full audit of ``local_path`` is cached at. - Reads through the store and key :func:`run_sieve_audit` writes to, so an - operator's ``[stores.cache]`` is honoured on both sides. A cache from - another framework, another level or a filtered run is a miss. + The store and key :func:`run_sieve_audit` writes to, so an operator's + ``[stores.cache]`` is honoured by the read and by the invalidation alike. + The caller closes the bundle. """ - from darnit.core.audit_cache import read_audit_cache from darnit.stores.selection import resolve_stores if operator_config is None: @@ -225,20 +216,56 @@ def read_full_audit_cache( operator_config = resolve_operator_config(local_path) stores_config = _load_merged_stores(framework_name, operator_config.config) stores_bundle = resolve_stores(stores_config, repo_path=Path(local_path)) + return stores_bundle, audit_cache_key(local_path, stores_config) + + +def read_full_audit_cache( + local_path: str, + framework_name: str, + level: int, + operator_config: "LoadedOperatorConfig | None" = None, +) -> dict[str, Any] | None: + """Return the cached full audit of ``framework_name`` at ``level``, or None. + + A cache from another framework or another level is a miss. A filtered + audit is never cached, so it cannot be returned here. + """ + from darnit.core.audit_cache import read_audit_cache + + stores_bundle, cache_key = _full_audit_cache_location(local_path, framework_name, operator_config) try: return read_audit_cache( local_path, expected_framework=framework_name, expected_level=level, - expected_tags="", - expected_filtered=False, store=stores_bundle.cache, - cache_key=audit_cache_key(local_path, stores_config, filtered=False), + cache_key=cache_key, ) finally: stores_bundle.close_all() +def invalidate_full_audit_cache( + local_path: str, + framework_name: str, + operator_config: "LoadedOperatorConfig | None" = None, +) -> None: + """Expire the cached full audit, in the place :func:`read_full_audit_cache` reads it. + + Remediation calls this after it changed something. With a configured + ``[stores.cache]``, expiring the default store's entry would leave the + configured one in place, and the next remediation would act on the audit + from before the change. + """ + from darnit.core.audit_cache import invalidate_audit_cache + + stores_bundle, cache_key = _full_audit_cache_location(local_path, framework_name, operator_config) + try: + invalidate_audit_cache(local_path, store=stores_bundle.cache, cache_key=cache_key) + finally: + stores_bundle.close_all() + + def _load_merged_mcp_servers(framework_name: str | None, operator: "OperatorConfig | None" = None) -> dict[str, Any]: """Return the merged ``mcp_servers`` allowlist for this audit run. @@ -803,7 +830,6 @@ def run_sieve_audit( *, controls: list | None = None, tags: list[str] | None = None, - cache_tags: str | None = None, cache_filtered: bool | None = None, evaluate_claims: bool = True, stop_on_llm: bool = True, @@ -849,6 +875,9 @@ def run_sieve_audit( write_cache: Write the results to the audit cache. A subset run, such as remediation's re-check (feature 043), passes False so it does not replace the full-audit cache. + cache_filtered: Whether ``controls`` is narrower than the framework's + full set. A filtered audit is not cached (issue #542). When unset, + a caller that passes ``controls`` is treated as filtered. Returns: Tuple of (results, summary) where results is a list of check result @@ -1088,10 +1117,10 @@ def run_sieve_audit( # already encodes repo identity (uses fixed "audit-cache"); configured # store's root is operator-picked and may be shared across repos, so # the key must encode repo identity. - # Issue #542: record the scope this audit ran at, and keep a filtered run - # out of the full-audit entry. - cache_tags, cache_filtered = _cache_scope(tags, controls, cache_tags, cache_filtered) - cache_key = audit_cache_key(local_path, stores_config, filtered=cache_filtered) + # Issue #542: only an unfiltered audit is cached. A subset stored here + # would be read back by remediation as if it were the full audit. + cache_filtered = _is_filtered(tags, controls, cache_filtered) + cache_key = audit_cache_key(local_path, stores_config) from darnit.core.audit_cache import write_audit_cache @@ -1099,7 +1128,7 @@ def run_sieve_audit( # internally. The enclosing try/except is defensive belt-and-braces # against any unexpected exception path (e.g. TypeError from a # future signature change). - if write_cache: + if write_cache and not cache_filtered: try: write_audit_cache( local_path, @@ -1107,8 +1136,6 @@ def run_sieve_audit( summary, level, resolved_fw or "", - tags=cache_tags, - filtered=cache_filtered, store=stores_bundle.cache, cache_key=cache_key, ) diff --git a/tests/darnit/core/test_audit_cache.py b/tests/darnit/core/test_audit_cache.py index dba6368c..02571ed8 100644 --- a/tests/darnit/core/test_audit_cache.py +++ b/tests/darnit/core/test_audit_cache.py @@ -135,8 +135,6 @@ def test_envelope_structure(self, temp_git_repo: Path, sample_results, sample_su "commit_dirty", "level", "framework", - "tags", - "filtered", "results", "summary", } @@ -492,140 +490,141 @@ def test_level_mismatch_is_a_miss(self, temp_git_repo, sample_results, sample_su assert read_audit_cache(str(temp_git_repo), expected_level=1) is not None assert read_audit_cache(str(temp_git_repo), expected_level=3) is None - def test_tag_filtered_cache_does_not_satisfy_an_unfiltered_read( - self, temp_git_repo, sample_results, sample_summary - ): - write_audit_cache( - str(temp_git_repo), - sample_results, - sample_summary, - 3, - "openssf-baseline", - tags="domain=VM", - ) - assert read_audit_cache(str(temp_git_repo), expected_tags="domain=VM") is not None - assert read_audit_cache(str(temp_git_repo), expected_tags="") is None - def test_unscoped_read_still_hits(self, temp_git_repo, sample_results, sample_summary): """Existing callers that pass no expectations are unaffected.""" - write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 1, "amber", tags="domain=VM") + write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 1, "amber") assert read_audit_cache(str(temp_git_repo)) is not None -class TestCacheVersionAndFilteredFlag: - """Review on #549: v1 envelopes must miss, and `filtered` covers the - filters a tag string cannot express (--include/--exclude/profile).""" +class TestCacheVersion: + """Review on #549: an envelope written before the scope fix must miss. + + Version 1 envelopes could hold a filtered audit under the full-audit key. + """ def test_old_version_envelope_is_a_miss(self, temp_git_repo, sample_results, sample_summary): write_audit_cache(str(temp_git_repo), sample_results, sample_summary, 3, "openssf-baseline") cache_path = _get_cache_dir(str(temp_git_repo)) / CACHE_FILENAME data = json.loads(cache_path.read_text()) data["version"] = 1 - data.pop("tags", None) - data.pop("filtered", None) cache_path.write_text(json.dumps(data)) assert read_audit_cache(str(temp_git_repo)) is None - def test_filtered_cache_does_not_satisfy_an_unfiltered_read(self, temp_git_repo, sample_results, sample_summary): - write_audit_cache( - str(temp_git_repo), - sample_results, - sample_summary, - 3, - "openssf-baseline", - filtered=True, - ) - assert read_audit_cache(str(temp_git_repo), expected_filtered=True) is not None - assert read_audit_cache(str(temp_git_repo), expected_filtered=False) is None - - def test_filtered_is_independent_of_tags(self, temp_git_repo, sample_results, sample_summary): - """--include narrows the set without producing a tag string.""" - write_audit_cache( - str(temp_git_repo), - sample_results, - sample_summary, - 3, - "openssf-baseline", - tags="", - filtered=True, - ) - assert read_audit_cache(str(temp_git_repo), expected_tags="") is not None - assert read_audit_cache(str(temp_git_repo), expected_filtered=False) is None - -class TestCacheScopeDecision: - """What run_sieve_audit records as an audit's scope (#542, review on #549).""" +class TestFilteredDecision: + """Whether run_sieve_audit treats an audit as filtered, and so does not cache it (#542).""" @pytest.mark.parametrize( - ("tags", "controls", "cache_tags", "cache_filtered", "expected"), + ("tags", "controls", "cache_filtered", "expected"), [ - (None, None, None, None, ("", False)), # darnit loads the controls: a full audit - (["domain=VM"], None, None, None, ("domain=VM", True)), - (None, ["c"], None, None, ("", True)), # caller's own list, not declared: fail closed - (None, ["c"], None, False, ("", False)), # caller says it is the full set - (None, ["c"], None, True, ("", True)), # caller says it narrowed, e.g. a profile - (["level=1"], ["c"], None, False, ("level=1", True)), # tags win over "not filtered" - (None, ["c"], "domain=VM", False, ("domain=VM", True)), + (None, None, None, False), # darnit loads the controls: a full audit + (["domain=VM"], None, None, True), + (None, ["c"], None, True), # caller's own list, not declared: fail closed + (None, ["c"], False, False), # caller says it is the full set + (None, ["c"], True, True), # caller says it narrowed, e.g. a profile + (["level=1"], ["c"], False, True), # tags win over "not filtered" ], ) - def test_scope(self, tags, controls, cache_tags, cache_filtered, expected): - from darnit.tools.audit import _cache_scope + def test_is_filtered(self, tags, controls, cache_filtered, expected): + from darnit.tools.audit import _is_filtered - assert _cache_scope(tags, controls, cache_tags, cache_filtered) == expected + assert _is_filtered(tags, controls, cache_filtered) is expected - def test_filtered_audits_get_their_own_key(self): + def test_one_key_per_repository(self): from darnit.tools.audit import audit_cache_key - assert audit_cache_key("/repo", None, filtered=False) == "audit-cache" - assert audit_cache_key("/repo", None, filtered=True) == "audit-cache-filtered" + assert audit_cache_key("/repo", None) == "audit-cache" + + +def _write_full_audit(repo, results, summary, *, level=3, framework="openssf-baseline"): + """Write the way run_sieve_audit does: through the resolved store, under the shared key.""" + from darnit.stores.selection import resolve_stores + from darnit.tools.audit import audit_cache_key + + bundle = resolve_stores(None, repo_path=Path(repo)) + try: + write_audit_cache( + str(repo), + results, + summary, + level, + framework, + store=bundle.cache, + cache_key=audit_cache_key(str(repo), None), + ) + finally: + bundle.close_all() + + +def _read_full_audit(repo): + from darnit.tools.audit import read_full_audit_cache + + return read_full_audit_cache(str(repo), "openssf-baseline", level=3) class TestFullAuditCacheRead: """read_full_audit_cache is what remediation reads (#542).""" - @staticmethod - def _write(repo, results, summary, *, level=3, framework="openssf-baseline", filtered=False, tags=""): - # Write the way run_sieve_audit does: through the resolved store, - # under the shared key. + def test_full_audit_is_a_hit(self, temp_git_repo, sample_results, sample_summary): + _write_full_audit(temp_git_repo, sample_results, sample_summary) + assert _read_full_audit(temp_git_repo) is not None + + def test_level_1_audit_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + _write_full_audit(temp_git_repo, sample_results, sample_summary, level=1) + assert _read_full_audit(temp_git_repo) is None + + def test_other_framework_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + _write_full_audit(temp_git_repo, sample_results, sample_summary, framework="gittuf") + assert _read_full_audit(temp_git_repo) is None + + +class TestFullAuditCacheInvalidation: + """invalidate_full_audit_cache expires what read_full_audit_cache reads (review on #549).""" + + def test_invalidated_cache_is_a_miss(self, temp_git_repo, sample_results, sample_summary): + from darnit.tools.audit import invalidate_full_audit_cache + + _write_full_audit(temp_git_repo, sample_results, sample_summary) + assert _read_full_audit(temp_git_repo) is not None + invalidate_full_audit_cache(str(temp_git_repo), "openssf-baseline") + assert _read_full_audit(temp_git_repo) is None + + def test_configured_store_is_invalidated( + self, temp_git_repo, tmp_path_factory, monkeypatch, sample_results, sample_summary + ): + """Review on #549: with ``[stores.cache]`` set, the configured entry must expire. + + The plain ``invalidate_audit_cache(path)`` expires the default store's + entry and leaves the configured one, so remediation kept reading the + audit from before its own change. + """ + from darnit.config.framework_schema import StoreBlock, StoresConfig + from darnit.core.audit_cache import invalidate_audit_cache from darnit.stores.selection import resolve_stores - from darnit.tools.audit import audit_cache_key + from darnit.tools.audit import audit_cache_key, invalidate_full_audit_cache + + root = tmp_path_factory.mktemp("configured-cache") + stores = StoresConfig(cache=StoreBlock(backend="local-fs", root=str(root))) + monkeypatch.setattr("darnit.tools.audit._load_merged_stores", lambda *args, **kwargs: stores) - bundle = resolve_stores(None, repo_path=Path(repo)) + bundle = resolve_stores(stores, repo_path=Path(temp_git_repo)) try: write_audit_cache( - str(repo), results, summary, level, framework, - tags=tags, filtered=filtered, + str(temp_git_repo), + sample_results, + sample_summary, + 3, + "openssf-baseline", store=bundle.cache, - cache_key=audit_cache_key(str(repo), None, filtered=filtered), + cache_key=audit_cache_key(str(temp_git_repo), stores), ) finally: bundle.close_all() + assert list(root.glob("*.json")), "the audit should be cached in the configured store" + assert _read_full_audit(temp_git_repo) is not None - @staticmethod - def _read(repo): - from darnit.tools.audit import read_full_audit_cache - - return read_full_audit_cache(str(repo), "openssf-baseline", level=3) - - def test_full_audit_is_a_hit(self, temp_git_repo, sample_results, sample_summary): - self._write(temp_git_repo, sample_results, sample_summary) - assert self._read(temp_git_repo) is not None - - def test_filtered_audit_is_a_miss(self, temp_git_repo, sample_results, sample_summary): - self._write(temp_git_repo, sample_results, sample_summary, filtered=True) - assert self._read(temp_git_repo) is None - - def test_filtered_audit_does_not_replace_the_full_one(self, temp_git_repo, sample_results, sample_summary): - self._write(temp_git_repo, sample_results, sample_summary) - self._write(temp_git_repo, [], {}, filtered=True, tags="domain=VM") - cache = self._read(temp_git_repo) - assert cache is not None - assert cache["results"] == sample_results - - def test_level_1_audit_is_a_miss(self, temp_git_repo, sample_results, sample_summary): - self._write(temp_git_repo, sample_results, sample_summary, level=1) - assert self._read(temp_git_repo) is None + invalidate_audit_cache(str(temp_git_repo)) + assert _read_full_audit(temp_git_repo) is not None, "the default-store call does not reach it" - def test_other_framework_is_a_miss(self, temp_git_repo, sample_results, sample_summary): - self._write(temp_git_repo, sample_results, sample_summary, framework="gittuf") - assert self._read(temp_git_repo) is None + invalidate_full_audit_cache(str(temp_git_repo), "openssf-baseline") + assert _read_full_audit(temp_git_repo) is None diff --git a/tests/darnit_baseline/test_audit_cache_scope.py b/tests/darnit_baseline/test_audit_cache_scope.py index 985505c0..a186c148 100644 --- a/tests/darnit_baseline/test_audit_cache_scope.py +++ b/tests/darnit_baseline/test_audit_cache_scope.py @@ -1,4 +1,4 @@ -"""The Baseline audit tool tells the cache when a profile narrowed the audit (#542).""" +"""A Baseline audit narrowed by a profile is not cached as a full audit (#542).""" from __future__ import annotations @@ -7,22 +7,23 @@ import pytest -@pytest.mark.parametrize(("profile", "expected"), [("level1_quick", True), (None, False)]) -def test_profile_audit_is_recorded_as_filtered(tmp_path, monkeypatch, profile, expected) -> None: - captured: dict = {} +def _repo_with_one_commit(path) -> None: + """A commit is needed: the cache of a repository with no HEAD is always stale.""" + git = ["git", "-C", str(path), "-c", "user.name=test", "-c", "user.email=test@example.com"] + subprocess.run(["git", "init", "-q", str(path)], check=True) + (path / "README.md").write_text("# test\n") + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "-q", "-m", "init"], check=True) - def recording_audit(**kwargs): - captured.update(kwargs) - return [], {} - - monkeypatch.setattr("darnit.tools.audit.run_sieve_audit", recording_audit) - subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) +@pytest.mark.parametrize(("profile", "cached"), [("level1_quick", False), (None, True)]) +def test_only_an_unfiltered_audit_is_cached(tmp_path, profile, cached) -> None: + from darnit.tools.audit import read_full_audit_cache from darnit_baseline.tools import audit_openssf_baseline - try: - audit_openssf_baseline(local_path=str(tmp_path), profile=profile, auto_init_config=False) - except Exception: # noqa: BLE001 - only the arguments handed to the audit matter here - pass + _repo_with_one_commit(tmp_path) + + audit_openssf_baseline(local_path=str(tmp_path), profile=profile, auto_init_config=False) - assert captured["cache_filtered"] is expected + cache = read_full_audit_cache(str(tmp_path), "openssf-baseline", level=3) + assert (cache is not None) is cached From 3f0a11453745d11e6506860ce65c126a20a133c3 Mon Sep 17 00:00:00 2001 From: Marc-cn <130138935+Marc-cn@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:53:27 -0400 Subject: [PATCH 3/3] test(cli): a filtered darnit audit leaves no full-audit cache Review on #549: nothing tested the count rule in cli.py. A plain audit is cached; -t, --include and --exclude runs are not. Also fix a comment that still named the removed envelope keys. Assisted-by: Claude:claude-fable-5-1 Signed-off-by: Marc-cn <130138935+Marc-cn@users.noreply.github.com> --- .../darnit/src/darnit/core/audit_cache.py | 4 +- tests/darnit/test_cli_audit_cache.py | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 tests/darnit/test_cli_audit_cache.py diff --git a/packages/darnit/src/darnit/core/audit_cache.py b/packages/darnit/src/darnit/core/audit_cache.py index 3e7891d8..88dd70e4 100644 --- a/packages/darnit/src/darnit/core/audit_cache.py +++ b/packages/darnit/src/darnit/core/audit_cache.py @@ -269,8 +269,8 @@ def read_audit_cache( # Version check version = data.get("version") # Issue #542 review: an exact match, not an upper bound. A v1 envelope - # has no `tags`/`filtered` keys, so reading it would present a possibly - # filtered audit as unfiltered. Older caches miss and are re-audited. + # may hold a filtered audit under the full-audit key, so reading it would + # present a subset as the full audit. Older caches miss and are re-audited. if not isinstance(version, int) or version != CACHE_VERSION: logger.debug("Unknown audit cache version: %s", version) return None diff --git a/tests/darnit/test_cli_audit_cache.py b/tests/darnit/test_cli_audit_cache.py new file mode 100644 index 00000000..74400d1c --- /dev/null +++ b/tests/darnit/test_cli_audit_cache.py @@ -0,0 +1,39 @@ +"""Only a plain `darnit audit` leaves a full-audit cache (#542).""" + +from __future__ import annotations + +import subprocess + +import pytest + +from darnit.cli import main + + +def _repo_with_one_commit(path) -> None: + """A commit is needed: the cache of a repository with no HEAD is always stale.""" + git = ["git", "-C", str(path), "-c", "user.name=test", "-c", "user.email=test@example.com"] + subprocess.run(["git", "init", "-q", str(path)], check=True) + (path / "README.md").write_text("# test\n") + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "-q", "-m", "init"], check=True) + + +@pytest.mark.parametrize( + ("flags", "cached"), + [ + ([], True), + (["-t", "level=1"], False), + (["--include", "OSPS-QA-05.01"], False), + (["--exclude", "OSPS-QA-05.01"], False), + ], +) +def test_only_a_plain_audit_is_cached(tmp_path, flags, cached) -> None: + from darnit.tools.audit import read_full_audit_cache + + _repo_with_one_commit(tmp_path) + + code = main(["audit", "-f", "openssf-baseline", "--no-fail", *flags, str(tmp_path)]) + + assert code == 0 + cache = read_full_audit_cache(str(tmp_path), "openssf-baseline", level=3) + assert (cache is not None) is cached