diff --git a/examples/cases/ctf-lictor-fmtstr-001.json b/examples/cases/ctf-lictor-fmtstr-001.json index 56efb74..6fea71a 100644 --- a/examples/cases/ctf-lictor-fmtstr-001.json +++ b/examples/cases/ctf-lictor-fmtstr-001.json @@ -15,6 +15,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "FLAG{the_format_is_the_weapon}" + "expected_flag": "FLAG{the_format_is_the_weapon}", + "expected_technique": "fmtstr" } } diff --git a/examples/cases/ctf-lictor-orw-001.json b/examples/cases/ctf-lictor-orw-001.json index 2b634ba..d475473 100644 --- a/examples/cases/ctf-lictor-orw-001.json +++ b/examples/cases/ctf-lictor-orw-001.json @@ -15,6 +15,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "FLAG{orw_reads_the_flag}" + "expected_flag": "FLAG{orw_reads_the_flag}", + "expected_technique": "orw" } } diff --git a/examples/cases/ctf-lictor-ret2func-001.json b/examples/cases/ctf-lictor-ret2func-001.json index d102728..ed50b9a 100644 --- a/examples/cases/ctf-lictor-ret2func-001.json +++ b/examples/cases/ctf-lictor-ret2func-001.json @@ -15,6 +15,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "FLAG{augure_decides_lictor_executes}" + "expected_flag": "FLAG{augure_decides_lictor_executes}", + "expected_technique": "ret2func" } } diff --git a/examples/cases/ctf-lictor-ret2plt-001.json b/examples/cases/ctf-lictor-ret2plt-001.json index 07304f0..bc10eb4 100644 --- a/examples/cases/ctf-lictor-ret2plt-001.json +++ b/examples/cases/ctf-lictor-ret2plt-001.json @@ -15,6 +15,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "FLAG{the_data_decides_ret2plt}" + "expected_flag": "FLAG{the_data_decides_ret2plt}", + "expected_technique": "ret2plt" } } diff --git a/examples/cases/ctf-lictor-shellcode-001.json b/examples/cases/ctf-lictor-shellcode-001.json index f6de87d..6adf829 100644 --- a/examples/cases/ctf-lictor-shellcode-001.json +++ b/examples/cases/ctf-lictor-shellcode-001.json @@ -15,6 +15,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "FLAG{nx_off_says_shellcode}" + "expected_flag": "FLAG{nx_off_says_shellcode}", + "expected_technique": "shellcode" } } diff --git a/examples/cases/ctf-lictor-uaf_heap-001.json b/examples/cases/ctf-lictor-uaf_heap-001.json index fd0987a..b7432d2 100644 --- a/examples/cases/ctf-lictor-uaf_heap-001.json +++ b/examples/cases/ctf-lictor-uaf_heap-001.json @@ -15,6 +15,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "FLAG{tcache_reuse_wins}" + "expected_flag": "FLAG{tcache_reuse_wins}", + "expected_technique": "uaf_heap" } } diff --git a/examples/cases/ctf-rop-badchars-001.json b/examples/cases/ctf-rop-badchars-001.json index ab1efb2..c910ce8 100644 --- a/examples/cases/ctf-rop-badchars-001.json +++ b/examples/cases/ctf-rop-badchars-001.json @@ -16,6 +16,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "ROPE{a_placeholder_32byte_flag!}" + "expected_flag": "ROPE{a_placeholder_32byte_flag!}", + "expected_technique": "badchars" } } diff --git a/examples/cases/ctf-rop-fluff-001.json b/examples/cases/ctf-rop-fluff-001.json index 3a417cf..1d04b16 100644 --- a/examples/cases/ctf-rop-fluff-001.json +++ b/examples/cases/ctf-rop-fluff-001.json @@ -16,6 +16,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "ROPE{a_placeholder_32byte_flag!}" + "expected_flag": "ROPE{a_placeholder_32byte_flag!}", + "expected_technique": "fluff" } } diff --git a/examples/cases/ctf-rop-pivot-001.json b/examples/cases/ctf-rop-pivot-001.json index 856cb43..bc3f8ae 100644 --- a/examples/cases/ctf-rop-pivot-001.json +++ b/examples/cases/ctf-rop-pivot-001.json @@ -16,6 +16,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "ROPE{a_placeholder_32byte_flag!}" + "expected_flag": "ROPE{a_placeholder_32byte_flag!}", + "expected_technique": "pivot" } } diff --git a/examples/cases/ctf-rop-ret2win-001.json b/examples/cases/ctf-rop-ret2win-001.json index 0538338..cee3530 100644 --- a/examples/cases/ctf-rop-ret2win-001.json +++ b/examples/cases/ctf-rop-ret2win-001.json @@ -16,6 +16,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "ROPE{a_placeholder_32byte_flag!}" + "expected_flag": "ROPE{a_placeholder_32byte_flag!}", + "expected_technique": "ret2win" } } diff --git a/examples/cases/ctf-rop-split-001.json b/examples/cases/ctf-rop-split-001.json index 75e3b29..2b61a84 100644 --- a/examples/cases/ctf-rop-split-001.json +++ b/examples/cases/ctf-rop-split-001.json @@ -16,6 +16,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "ROPE{a_placeholder_32byte_flag!}" + "expected_flag": "ROPE{a_placeholder_32byte_flag!}", + "expected_technique": "split" } } diff --git a/examples/cases/ctf-rop-write4-001.json b/examples/cases/ctf-rop-write4-001.json index 1befe5f..fee7abe 100644 --- a/examples/cases/ctf-rop-write4-001.json +++ b/examples/cases/ctf-rop-write4-001.json @@ -16,6 +16,7 @@ "language": "c", "category": "pwn", "tier": "verified", - "expected_flag": "ROPE{a_placeholder_32byte_flag!}" + "expected_flag": "ROPE{a_placeholder_32byte_flag!}", + "expected_technique": "write4" } } diff --git a/experiments/ctf_ingest/ingest.py b/experiments/ctf_ingest/ingest.py index e138096..6115918 100644 --- a/experiments/ctf_ingest/ingest.py +++ b/experiments/ctf_ingest/ingest.py @@ -141,12 +141,14 @@ def lictor_cases(corpus: Path) -> list[tuple[dict, dict]]: "category": "pwn", "tier": "verified", "expected_flag": flag_value, + "expected_technique": target.name, }, } manifest = { "id": case_id, "tier": "verified", "expected_flag": bool(flag_value), + "expected_technique": True, "url": "https://github.com/log0u7/lictor", } cases.append((case, manifest)) @@ -187,12 +189,14 @@ def rop_emporium_cases(corpus: Path) -> list[tuple[dict, dict]]: "category": "pwn", "tier": "verified", "expected_flag": flag_value, + "expected_technique": challenge.name, }, } manifest = { "id": case_id, "tier": "verified", "expected_flag": bool(flag_value), + "expected_technique": challenge.name, "url": f"https://ropemporium.com/challenge/{challenge.name}.html", } cases.append((case, manifest)) diff --git a/experiments/ctf_ingest/manifest.jsonl b/experiments/ctf_ingest/manifest.jsonl index ff4d055..7a89c1f 100644 --- a/experiments/ctf_ingest/manifest.jsonl +++ b/experiments/ctf_ingest/manifest.jsonl @@ -1,15 +1,15 @@ -{"source": "lictor-lab", "id": "ctf-lictor-fmtstr-001", "tier": "verified", "expected_flag": true, "url": "https://github.com/log0u7/lictor"} -{"source": "lictor-lab", "id": "ctf-lictor-orw-001", "tier": "verified", "expected_flag": true, "url": "https://github.com/log0u7/lictor"} -{"source": "lictor-lab", "id": "ctf-lictor-ret2func-001", "tier": "verified", "expected_flag": true, "url": "https://github.com/log0u7/lictor"} -{"source": "lictor-lab", "id": "ctf-lictor-ret2plt-001", "tier": "verified", "expected_flag": true, "url": "https://github.com/log0u7/lictor"} -{"source": "lictor-lab", "id": "ctf-lictor-shellcode-001", "tier": "verified", "expected_flag": true, "url": "https://github.com/log0u7/lictor"} -{"source": "lictor-lab", "id": "ctf-lictor-uaf_heap-001", "tier": "verified", "expected_flag": true, "url": "https://github.com/log0u7/lictor"} -{"source": "rop-emporium", "id": "ctf-rop-badchars-001", "tier": "verified", "expected_flag": true, "url": "https://ropemporium.com/challenge/badchars.html"} -{"source": "rop-emporium", "id": "ctf-rop-fluff-001", "tier": "verified", "expected_flag": true, "url": "https://ropemporium.com/challenge/fluff.html"} -{"source": "rop-emporium", "id": "ctf-rop-pivot-001", "tier": "verified", "expected_flag": true, "url": "https://ropemporium.com/challenge/pivot.html"} -{"source": "rop-emporium", "id": "ctf-rop-ret2win-001", "tier": "verified", "expected_flag": true, "url": "https://ropemporium.com/challenge/ret2win.html"} -{"source": "rop-emporium", "id": "ctf-rop-split-001", "tier": "verified", "expected_flag": true, "url": "https://ropemporium.com/challenge/split.html"} -{"source": "rop-emporium", "id": "ctf-rop-write4-001", "tier": "verified", "expected_flag": true, "url": "https://ropemporium.com/challenge/write4.html"} +{"source": "lictor-lab", "id": "ctf-lictor-fmtstr-001", "tier": "verified", "expected_flag": true, "expected_technique": true, "url": "https://github.com/log0u7/lictor"} +{"source": "lictor-lab", "id": "ctf-lictor-orw-001", "tier": "verified", "expected_flag": true, "expected_technique": true, "url": "https://github.com/log0u7/lictor"} +{"source": "lictor-lab", "id": "ctf-lictor-ret2func-001", "tier": "verified", "expected_flag": true, "expected_technique": true, "url": "https://github.com/log0u7/lictor"} +{"source": "lictor-lab", "id": "ctf-lictor-ret2plt-001", "tier": "verified", "expected_flag": true, "expected_technique": true, "url": "https://github.com/log0u7/lictor"} +{"source": "lictor-lab", "id": "ctf-lictor-shellcode-001", "tier": "verified", "expected_flag": true, "expected_technique": true, "url": "https://github.com/log0u7/lictor"} +{"source": "lictor-lab", "id": "ctf-lictor-uaf_heap-001", "tier": "verified", "expected_flag": true, "expected_technique": true, "url": "https://github.com/log0u7/lictor"} +{"source": "rop-emporium", "id": "ctf-rop-badchars-001", "tier": "verified", "expected_flag": true, "expected_technique": "badchars", "url": "https://ropemporium.com/challenge/badchars.html"} +{"source": "rop-emporium", "id": "ctf-rop-fluff-001", "tier": "verified", "expected_flag": true, "expected_technique": "fluff", "url": "https://ropemporium.com/challenge/fluff.html"} +{"source": "rop-emporium", "id": "ctf-rop-pivot-001", "tier": "verified", "expected_flag": true, "expected_technique": "pivot", "url": "https://ropemporium.com/challenge/pivot.html"} +{"source": "rop-emporium", "id": "ctf-rop-ret2win-001", "tier": "verified", "expected_flag": true, "expected_technique": "ret2win", "url": "https://ropemporium.com/challenge/ret2win.html"} +{"source": "rop-emporium", "id": "ctf-rop-split-001", "tier": "verified", "expected_flag": true, "expected_technique": "split", "url": "https://ropemporium.com/challenge/split.html"} +{"source": "rop-emporium", "id": "ctf-rop-write4-001", "tier": "verified", "expected_flag": true, "expected_technique": "write4", "url": "https://ropemporium.com/challenge/write4.html"} {"source": "ctf-archives", "id": "ctf-zeekr-adas-fusion-spoof-001", "tier": "unverified", "expected_flag": false, "url": "https://github.com/CTF-Archives/2026-Zeekr-Cybersecurity-Quals/releases/tag/attachments"} {"source": "ctf-archives", "id": "ctf-zeekr-can-noir-001", "tier": "unverified", "expected_flag": false, "url": "https://github.com/CTF-Archives/2026-Zeekr-Cybersecurity-Quals/releases/tag/attachments"} {"source": "ctf-archives", "id": "ctf-zeekr-chromatic-custody-001", "tier": "unverified", "expected_flag": false, "url": "https://github.com/CTF-Archives/2026-Zeekr-Cybersecurity-Quals/releases/tag/attachments"} diff --git a/src/brainforge/domains/ctf/pack.py b/src/brainforge/domains/ctf/pack.py index c773fed..fb53ef9 100644 --- a/src/brainforge/domains/ctf/pack.py +++ b/src/brainforge/domains/ctf/pack.py @@ -118,32 +118,46 @@ def quality_gate( if kind == RoleKind.TEACHER: reasons.extend(self._teacher_checks(data)) elif kind == RoleKind.JUDGE: - reasons.extend(self._judge_checks(data)) - reasons.extend(self._flag_checks(data, case)) + expected_flag = str(case.metadata.get("expected_flag") or "").strip() if case else "" + # on verified-tier cases the flag lives outside the challenge + # material: the known-flag rules replace the generic demand + reasons.extend(self._judge_checks(data, require_flag=not expected_flag)) + reasons.extend(self._verification_checks(data, case)) if not data.get("summary") and not data.get("reasoning"): reasons.append("missing summary or reasoning") return GateResult(passed=not reasons, reasons=reasons) - def _flag_checks(self, data: dict, case: "Case | None") -> list[str]: - """Verify the judge verdict against a known flag (verified tier). + def _verification_checks(self, data: dict, case: "Case | None") -> list[str]: + """Verify the judge output against curated ground truth (verified tier). - Cases carrying ``metadata.expected_flag`` state the ground truth: - any non-empty flag that differs from it rejects the record - whatever the verdict, and a matching flag with a non-solved - verdict is a contradiction that rejects too. Cases without it - (tier B) skip the check entirely. + ``metadata.expected_technique`` (list of acceptable keywords) gates + the named technique: the analysis must identify it, whatever the + prose. ``metadata.expected_flag`` anti-hallucinates: a judge that + claims a flag must produce the known one, but an absent flag is + not an error (the flag lives outside the challenge material and + stays the RL reward's business). Cases without this metadata + (tier B) skip the checks entirely. """ reasons: list[str] = [] - expected = str(case.metadata.get("expected_flag") or "").strip() if case else "" - if not expected: + if case is None: return reasons + expected_technique = case.metadata.get("expected_technique") + if isinstance(expected_technique, str): + expected_technique = [expected_technique] + if isinstance(expected_technique, list) and expected_technique: + technique = str(data.get("technique") or "").lower() + keywords = [k.lower() for k in expected_technique if isinstance(k, str)] + if keywords and not any(keyword in technique for keyword in keywords): + reasons.append( + f"technique does not match expected_technique ({'/'.join(keywords)})" + ) + expected_flag_raw = case.metadata.get("expected_flag") + expected_flag = expected_flag_raw.strip() if isinstance(expected_flag_raw, str) else "" flag = str(data.get("flag") or "").strip() - if flag and flag != expected: + if expected_flag and flag and flag != expected_flag: reasons.append( "flag does not match expected_flag (verified tier; ground truth is known)" ) - if flag == expected and flag and data.get("verdict") != "solved": - reasons.append("flag matches expected_flag but the verdict is not solved") return reasons def _teacher_checks(self, data: dict) -> list[str]: @@ -156,10 +170,10 @@ def _teacher_checks(self, data: dict) -> list[str]: reasons.append("missing reasoning") return reasons - def _judge_checks(self, data: dict) -> list[str]: + def _judge_checks(self, data: dict, require_flag: bool = True) -> list[str]: reasons = [] if data.get("verdict") == "solved": - if not data.get("flag"): + if require_flag and not data.get("flag"): reasons.append("solved verdict without flag") if not data.get("exploit_path"): reasons.append("solved verdict without exploit_path") diff --git a/tests/unit/test_ctf_pack.py b/tests/unit/test_ctf_pack.py index 2f6d126..ddf4bf9 100644 --- a/tests/unit/test_ctf_pack.py +++ b/tests/unit/test_ctf_pack.py @@ -217,70 +217,88 @@ def _judge_payload(flag="FLAG{known}"): } -def test_judge_gate_rejects_flag_mismatch_against_expected_flag(ctf_case): +def _judge_payload(flag="FLAG{known}", verdict="solved", technique="pwn/buffer-overflow ret2win"): + return { + "verdict": verdict, + "technique": technique, + "exploit_path": ["overflow", "ret2win"], + "flag": flag, + "writeup": "Overflow the buffer and jump to win().", + "confidence": 0.9, + "reasoning": "The binary prints the flag from win().", + "disagreement": False, + } + + +def test_judge_gate_passes_matching_expected_flag(ctf_case): pack = get_pack("ctf") verified = ctf_case.model_copy(update={"metadata": {"expected_flag": "FLAG{known}"}}) - result = pack.quality_gate( - RoleKind.JUDGE, _judge_payload(flag="FLAG{wrong}"), min_confidence=0.5, case=verified - ) - assert not result.passed - assert any("expected_flag" in reason for reason in result.reasons) + result = pack.quality_gate(RoleKind.JUDGE, _judge_payload(), min_confidence=0.5, case=verified) + assert result.passed -def test_judge_gate_accepts_matching_expected_flag(ctf_case): +def test_judge_gate_accepts_technique_keyword_inside_prose(ctf_case): pack = get_pack("ctf") - verified = ctf_case.model_copy(update={"metadata": {"expected_flag": "FLAG{known}"}}) + verified = ctf_case.model_copy( + update={"metadata": {"expected_technique": ["ret2func", "ret2win"]}} + ) result = pack.quality_gate(RoleKind.JUDGE, _judge_payload(), min_confidence=0.5, case=verified) assert result.passed -def test_judge_gate_rejects_flag_match_with_unsolved_verdict(ctf_case): +def test_judge_gate_rejects_wrong_technique(ctf_case): pack = get_pack("ctf") - verified = ctf_case.model_copy(update={"metadata": {"expected_flag": "FLAG{known}"}}) - data = _judge_payload() - data["verdict"] = "insufficient_information" - result = pack.quality_gate(RoleKind.JUDGE, data, min_confidence=0.5, case=verified) + verified = ctf_case.model_copy( + update={"metadata": {"expected_technique": ["fmtstr", "format string"]}} + ) + result = pack.quality_gate( + RoleKind.JUDGE, + _judge_payload(technique="pwn/heap overflow tcache"), + min_confidence=0.5, + case=verified, + ) assert not result.passed - assert any("verdict" in reason for reason in result.reasons) + assert any("expected_technique" in reason for reason in result.reasons) -def test_judge_gate_unaffected_without_expected_flag(ctf_case): +def test_judge_gate_rejects_hallucinated_flag(ctf_case): pack = get_pack("ctf") - result = pack.quality_gate(RoleKind.JUDGE, _judge_payload(), min_confidence=0.5, case=ctf_case) - assert result.passed + verified = ctf_case.model_copy(update={"metadata": {"expected_flag": "FLAG{known}"}}) + result = pack.quality_gate( + RoleKind.JUDGE, _judge_payload(flag="FLAG{invented}"), min_confidence=0.5, case=verified + ) + assert not result.passed + assert any("expected_flag" in reason for reason in result.reasons) -def test_judge_gate_flag_comparison_strips_whitespace(ctf_case): +def test_judge_gate_flag_absence_is_not_an_error(ctf_case): pack = get_pack("ctf") - verified = ctf_case.model_copy(update={"metadata": {"expected_flag": "FLAG{known} "}}) + verified = ctf_case.model_copy(update={"metadata": {"expected_flag": "FLAG{known}"}}) result = pack.quality_gate( - RoleKind.JUDGE, _judge_payload(flag=" FLAG{known}\n"), min_confidence=0.5, case=verified + RoleKind.JUDGE, _judge_payload(flag=None), min_confidence=0.5, case=verified ) assert result.passed -def test_judge_gate_non_string_expected_flag_does_not_crash(ctf_case): +def test_judge_gate_tier_b_unaffected(ctf_case): pack = get_pack("ctf") - verified = ctf_case.model_copy(update={"metadata": {"expected_flag": 12345}}) - result = pack.quality_gate(RoleKind.JUDGE, _judge_payload(), min_confidence=0.5, case=verified) - assert not result.passed - assert any("expected_flag" in reason for reason in result.reasons) + result = pack.quality_gate(RoleKind.JUDGE, _judge_payload(), min_confidence=0.5, case=ctf_case) + assert result.passed -def test_judge_gate_rejects_wrong_flag_on_partial_verdict(ctf_case): +def test_judge_gate_non_string_expected_metadata_does_not_crash(ctf_case): pack = get_pack("ctf") - verified = ctf_case.model_copy(update={"metadata": {"expected_flag": "FLAG{known}"}}) - data = _judge_payload(flag="FLAG{wrong}") - data["verdict"] = "partial" - result = pack.quality_gate(RoleKind.JUDGE, data, min_confidence=0.5, case=verified) - assert not result.passed - assert any("expected_flag" in reason for reason in result.reasons) + verified = ctf_case.model_copy( + update={"metadata": {"expected_flag": 12345, "expected_technique": 99}} + ) + result = pack.quality_gate(RoleKind.JUDGE, _judge_payload(), min_confidence=0.5, case=verified) + assert result.passed -def test_judge_gate_partial_with_no_flag_stays_gated_on_verdict_only(ctf_case): +def test_judge_gate_flag_comparison_strips_whitespace(ctf_case): pack = get_pack("ctf") - verified = ctf_case.model_copy(update={"metadata": {"expected_flag": "FLAG{known}"}}) - data = _judge_payload(flag=None) - data["verdict"] = "partial" - result = pack.quality_gate(RoleKind.JUDGE, data, min_confidence=0.5, case=verified) + verified = ctf_case.model_copy(update={"metadata": {"expected_flag": "FLAG{known} "}}) + result = pack.quality_gate( + RoleKind.JUDGE, _judge_payload(flag=" FLAG{known}\n"), min_confidence=0.5, case=verified + ) assert result.passed