From 8e648f624e888548fbb4f9fa245aa447b97b5e25 Mon Sep 17 00:00:00 2001 From: "kapil.madan" Date: Sun, 30 Aug 2026 16:59:03 +0530 Subject: [PATCH] feat: an extended four-act demo, and quieten the noise it exposed GOPAL's demo is one beat because GOPAL decides things: swap the model, the build fails. AICertify gathers things, and the interesting thing about it is how honest it is about the part it cannot gather. That needs four acts. 1. score-card bert-base-uncased a real card, scored, no setup 2. explain eu_ai_act 150 you declare, 14 an evaluator computes 3. init-contract + evaluate 29 policies, 0 green, because you have asserted none of the 150 4. metric gap report 22 of 27 supplied, and the five that are not, which is a contributor's list Two artefacts from one recorder: a 48-second CSS-animated SVG for the README, where JavaScript never runs, and the full 93-second cast for the website. Commands are really run and their output captured verbatim. Only the timing is synthesised, because the four take about three minutes between them and a cast at real speed is unwatchable. The recorder refuses to write if bert stops scoring 0.49, if explain stops reporting 150 and 14, if an empty contract stops failing all 29 policies, or if the gap report stops saying 22 of 27. Two fixes the recording forced out: extraction.py logged an expected condition at error and dumped a pydantic validation report per policy, so a successful evaluation looked like a crash. The schema path only recognises report_output, which four of gopal's policies define; every other policy fails it and the caller then reads the decision rules instead. The #78 fallback made that correct and left the log level behind. The highlighter counted lines where it should have counted screen rows. Two 130-character warnings wrapped, and `BELOW THRESHOLD` was drawn across an unrelated score three rows below itself. Verified now by replaying the cast through a VT emulator and checking every highlighted row against what the command actually printed. Pins gopal 0936496, which is the version the numbers on screen come from. --- README.md | 11 + aicertify/opa_core/extraction.py | 17 +- aicertify/opa_policies | 2 +- docs/demo/aicertify-animated.svg | 407 +++++++++++++++++++++++ docs/demo/aicertify-short.cast | 264 +++++++++++++++ docs/demo/aicertify.cast | 534 +++++++++++++++++++++++++++++++ scripts/record-demo.py | 415 ++++++++++++++++++++++++ 7 files changed, 1645 insertions(+), 5 deletions(-) create mode 100644 docs/demo/aicertify-animated.svg create mode 100644 docs/demo/aicertify-short.cast create mode 100644 docs/demo/aicertify.cast create mode 100644 scripts/record-demo.py diff --git a/README.md b/README.md index 6f68fa2..7bddd77 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,17 @@
+

+ A terminal session. aicertify score-card bert-base-uncased reports completeness 0.49 against a threshold of 0.8 and prints BELOW THRESHOLD, with a per-section bar chart. aicertify explain eu_ai_act then reports 150 fields you must declare, which no evaluator can observe, and 14 fields produced by evaluators, noting that asserting your own fairness score defeats the point. +

+ +

+ Scoring a real Hugging Face card, then asking what the EU AI Act actually needs.
+ Watch the full four-minute walkthrough, including running a contract and the metrics still missing an evaluator. +

+ +
+ **AICertify is the open execution and evidence layer for AI governance.** Describe an AI system in a contract, supply the facts only your organisation can know, attach or compute measured metrics, evaluate that evidence against versioned [GOPAL](https://github.com/Principled-Evolution/gopal) policies through [Open Policy Agent](https://www.openpolicyagent.org/), and generate dated PDF, Markdown, JSON, or HTML reports. **The goal is simple: move from “we have an AI policy” to evidence another engineer, auditor, or risk team can inspect and reproduce.** diff --git a/aicertify/opa_core/extraction.py b/aicertify/opa_core/extraction.py index 1cd2dcc..f8d96ef 100644 --- a/aicertify/opa_core/extraction.py +++ b/aicertify/opa_core/extraction.py @@ -359,10 +359,19 @@ def extract_policy_results_with_schema( return extracted_policies except Exception as e: - logger.error(f"Error validating OPA results against schema: {e}") - # Dont try to recover using a more flexible approach - logger.warning( - "Failed to validate OPA results against schema, returning empty list" + # Expected, and the common case. This path validates against the + # `report_output` schema, which four of gopal's policies define. Every + # other policy fails it, and the caller then reads the decision rules + # instead (see the note on extract_results_from_packages below). + # + # Logged at error until the #78 fallback landed, which meant a normal + # successful evaluation printed a pydantic validation report per policy + # and looked to a first-time user like a crash. The behaviour was + # already correct; only the log level was left behind. + logger.debug( + "No report_output schema match (%s); the caller falls back to " + "decision rules.", + e.__class__.__name__, ) return [] diff --git a/aicertify/opa_policies b/aicertify/opa_policies index 4321e7a..0936496 160000 --- a/aicertify/opa_policies +++ b/aicertify/opa_policies @@ -1 +1 @@ -Subproject commit 4321e7a4fc28061610d345319d6c778e877bbd85 +Subproject commit 09364967d494b4137b8c492135c9c7738964e601 diff --git a/docs/demo/aicertify-animated.svg b/docs/demo/aicertify-animated.svg new file mode 100644 index 0000000..5c4765e --- /dev/null +++ b/docs/demo/aicertify-animated.svg @@ -0,0 +1,407 @@ + +aicertify: 150 declared, 14 measured +$ aicertify score-card bert-base-uncased + +bert-base-uncased + completeness 0.49 (threshold 0.8) + quality 0.66 + BELOW THRESHOLD + + intended_use 1.00 #################### + training_data 0.67 ############# + caveats_recommendations 0.50 ########## + factors 0.50 ########## + metrics 0.50 ########## + quantitative_analyses 0.50 ########## + ethical_considerations 0.33 ####### + model_details 0.33 ###### + evaluation_data 0.00 + +A model card answers part of what Annex IV asks and then stops. +This is the documentation-completeness gap, not a compliance verdict. + +scored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0 +$ # so what does the EU AI Act actually ask for? +$ aicertify explain eu_ai_act | head -12 + +eu_ai_act — 29 policies + +Fields you must declare (150) + No evaluator can observe these. They are facts about your system, + your process, or your paperwork, so you assert them in the contract. + + accuracy.declared_in_instructions technical_robustness + accuracy.metrics_declared technical_robustness + assessment.completed conformity_assessment + assessment.harmonised_standards_applied conformity_assessment + assessment.notified_body_involved conformity_assessment +$ aicertify explain eu_ai_act | grep -A8 "produced by evaluators" +Fields produced by evaluators (14) + Computed at evaluation time from your interactions. Do not hand-write + these: asserting your own fairness score defeats the point. + + metrics.content_safety.score eu_fairness [content_safety evaluator] + metrics.emotion_recognition.combined_score emotion_recognition + metrics.emotion_recognition.educational_context emotion_recognition + metrics.emotion_recognition.workplace_context emotion_recognition + metrics.fairness.gender_bias eu_fairness [fairness evaluator] +$ + + + + + + +aicertify: 150 declared, 14 measured + + + + + + +$ +a +i +c +e +r +t +i +f +y +s +c +o +r +e +- +c +a +r +d +b +e +r +t +- +b +a +s +e +- +u +n +c +a +s +e +d +bert-base-uncased +c +o +m +p +l +e +t +e +n +e +s +s +0 +. +4 +9 +( +t +h +r +e +s +h +o +l +d +0 +. +8 +) + quality 0.66 +B +E +L +O +W +T +H +R +E +S +H +O +L +D + intended_use 1.00 #################### + training_data 0.67 ############# + caveats_recommendations 0.50 ########## + factors 0.50 ########## + metrics 0.50 ########## + quantitative_analyses 0.50 ########## + ethical_considerations 0.33 ####### + model_details 0.33 ###### + evaluation_data 0.00 +A model card answers part of what Annex IV asks and then stops. +This is the documentation-completeness gap, not a compliance verdict. +scored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0 +$ +# +s +o +w +h +a +t +d +o +e +s +t +h +e +E +U +A +I +A +c +t +a +c +t +u +a +l +k +l +y +a +s +k +f +o +r +? +$ +a +i +c +e +r +t +i +f +y +e +x +p +l +a +i +n +e +u +_ +a +i +_ +a +c +t +| +h +e +a +d +- +1 +2 +eu_ai_act — 29 policies +F +i +e +l +d +s +y +o +u +m +u +s +t +d +e +c +l +a +r +e +( +1 +5 +0 +) + No evaluator can observe these. They are facts about your system, + your process, or your paperwork, so you assert them in the contract. + accuracy.declared_in_instructions technical_robustness + accuracy.metrics_declared technical_robustness + assessment.completed conformity_assessment + assessment.harmonised_standards_applied conformity_assessment + assessment.notified_body_involved conformity_assessment +$ +a +i +c +e +r +t +i +f +y +e +x +p +l +a +i +n +e +u +_ +a +i +_ +a +c +t +| +g +r +e +p +- +A +8 +" +p +r +o +d +u +c +e +d +b +y +e +v +a +l +u +a +t +o +r +s +" +F +i +e +l +d +s +p +r +o +d +u +c +e +d +b +y +e +v +a +l +u +a +t +o +r +s +( +1 +4 +) + Computed at evaluation time from your interactions. Do not hand-write +t +h +e +s +e +: +a +s +s +e +r +t +i +n +g +y +o +u +r +o +w +n +f +a +i +r +n +e +s +s +s +c +o +r +e +d +e +f +e +a +t +s +t +h +e +p +o +i +n +t +. + metrics.content_safety.score eu_fairness [content_safety evaluator] + metrics.emotion_recognition.combined_score emotion_recognition + metrics.emotion_recognition.educational_context emotion_recognition + metrics.emotion_recognition.workplace_context emotion_recognition + metrics.fairness.gender_bias eu_fairness [fairness evaluator] +$ +$ █ + diff --git a/docs/demo/aicertify-short.cast b/docs/demo/aicertify-short.cast new file mode 100644 index 0000000..c827fe8 --- /dev/null +++ b/docs/demo/aicertify-short.cast @@ -0,0 +1,264 @@ +{"version": 2, "width": 100, "height": 30, "idle_time_limit": 3.0, "title": "AICertify: 150 declared, 14 measured"} +[0.6, "o", "$ "] +[0.6857, "o", "a"] +[0.7481, "o", "i"] +[0.8326, "o", "c"] +[0.8772, "o", "e"] +[0.9482, "o", "r"] +[0.993, "o", "t"] +[1.0405, "o", "i"] +[1.1202, "o", "f"] +[1.1796, "o", "y"] +[1.2503, "o", " "] +[1.3196, "o", "s"] +[1.3809, "o", "c"] +[1.45, "o", "o"] +[1.5289, "o", "r"] +[1.5857, "o", "e"] +[1.6409, "o", "-"] +[1.7185, "o", "c"] +[1.8039, "o", "a"] +[1.8679, "o", "r"] +[1.9161, "o", "d"] +[1.9636, "o", " "] +[2.0197, "o", "b"] +[2.0779, "o", "e"] +[2.1447, "o", "r"] +[2.2007, "o", "t"] +[2.2876, "o", "-"] +[2.3404, "o", "b"] +[2.4044, "o", "a"] +[2.4619, "o", "s"] +[2.5101, "o", "e"] +[2.5927, "o", "-"] +[2.661, "o", "u"] +[2.7296, "o", "n"] +[2.7831, "o", "c"] +[2.8522, "o", "a"] +[2.8905, "o", "s"] +[2.9325, "o", "e"] +[2.9761, "o", "d"] +[3.3961, "o", "\r\n"] +[5.4561, "o", "\r\nbert-base-uncased\r\n completeness 0.49 (threshold 0.8)\r\n"] +[5.5461, "o", " quality 0.66\r\n BELOW THRESHOLD\r\n\r\n"] +[5.6361, "o", " intended_use 1.00 ####################\r\n training_data 0.67 #############\r\n caveats_recommendations 0.50 ##########\r\n"] +[5.7261, "o", " factors 0.50 ##########\r\n metrics 0.50 ##########\r\n quantitative_analyses 0.50 ##########\r\n"] +[5.8161, "o", " ethical_considerations 0.33 #######\r\n model_details 0.33 ######\r\n evaluation_data 0.00 \r\n"] +[5.9061, "o", "\r\nA model card answers part of what Annex IV asks and then stops.\r\nThis is the documentation-completeness gap, not a compliance verdict.\r\n"] +[5.9961, "o", "\r\nscored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0\r\n"] +[6.8861, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0\u001b[0m\u001b8"] +[6.9141, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] +[6.9421, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] +[6.9701, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] +[7.3381, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] +[7.3661, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] +[7.3941, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] +[7.4221, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] +[7.4501, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] +[7.4781, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] +[7.5061, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] +[7.5341, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] +[7.5621, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] +[7.5901, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] +[7.6181, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] +[7.6461, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] +[7.6741, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] +[7.7021, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] +[7.7301, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] +[13.4981, "o", "$ "] +[13.5533, "o", "#"] +[13.6042, "o", " "] +[13.6675, "o", "s"] +[13.7216, "o", "o"] +[13.7668, "o", " "] +[13.8414, "o", "w"] +[13.901, "o", "h"] +[13.98, "o", "a"] +[14.036, "o", "t"] +[14.2622, "o", " "] +[14.3358, "o", "d"] +[14.407, "o", "o"] +[14.452, "o", "e"] +[14.5034, "o", "s"] +[14.656, "o", " "] +[14.7291, "o", "t"] +[14.8083, "o", "h"] +[14.8639, "o", "e"] +[14.9825, "o", " "] +[15.0553, "o", "E"] +[15.1175, "o", "U"] +[15.2782, "o", " "] +[15.3602, "o", "A"] +[15.4087, "o", "I"] +[15.4983, "o", " "] +[15.5678, "o", "A"] +[15.6207, "o", "c"] +[15.7021, "o", "t"] +[15.8514, "o", " "] +[15.9182, "o", "a"] +[15.9948, "o", "c"] +[16.0738, "o", "t"] +[16.1586, "o", "u"] +[16.2018, "o", "a"] +[16.2789, "o", "l"] +[16.3431, "o", "k"] +[16.7652, "o", "\b \b"] +[16.9292, "o", "l"] +[17.0206, "o", "y"] +[17.1123, "o", " "] +[17.1793, "o", "a"] +[17.2189, "o", "s"] +[17.2605, "o", "k"] +[17.304, "o", " "] +[17.3619, "o", "f"] +[17.432, "o", "o"] +[17.4911, "o", "r"] +[17.5342, "o", "?"] +[17.9542, "o", "\r\n"] +[18.6142, "o", "$ "] +[18.6746, "o", "a"] +[18.7332, "o", "i"] +[18.8196, "o", "c"] +[18.8938, "o", "e"] +[18.9528, "o", "r"] +[19.0121, "o", "t"] +[19.0838, "o", "i"] +[19.1662, "o", "f"] +[19.2485, "o", "y"] +[19.4441, "o", " "] +[19.5332, "o", "e"] +[19.625, "o", "x"] +[19.6876, "o", "p"] +[19.7399, "o", "l"] +[19.8257, "o", "a"] +[19.8648, "o", "i"] +[19.9051, "o", "n"] +[19.9678, "o", " "] +[20.0131, "o", "e"] +[20.075, "o", "u"] +[20.1434, "o", "_"] +[20.1838, "o", "a"] +[20.2341, "o", "i"] +[20.2763, "o", "_"] +[20.3213, "o", "a"] +[20.3978, "o", "c"] +[20.4606, "o", "t"] +[20.5349, "o", " "] +[20.587, "o", "|"] +[20.7452, "o", " "] +[20.7897, "o", "h"] +[20.8367, "o", "e"] +[20.8764, "o", "a"] +[20.9218, "o", "d"] +[20.9953, "o", " "] +[21.0738, "o", "-"] +[21.1456, "o", "1"] +[21.2333, "o", "2"] +[21.6533, "o", "\r\n"] +[23.7133, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] +[23.8033, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] +[23.8933, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] +[23.9833, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] +[24.7733, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] +[24.8013, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] +[24.8293, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] +[27.7973, "o", "$ "] +[27.8433, "o", "a"] +[27.9297, "o", "i"] +[28.0189, "o", "c"] +[28.07, "o", "e"] +[28.1519, "o", "r"] +[28.219, "o", "t"] +[28.2995, "o", "i"] +[28.3838, "o", "f"] +[28.4619, "o", "y"] +[28.519, "o", " "] +[28.6104, "o", "e"] +[28.6649, "o", "x"] +[28.7462, "o", "p"] +[28.7963, "o", "l"] +[28.8422, "o", "a"] +[28.9155, "o", "i"] +[28.9987, "o", "n"] +[29.1715, "o", " "] +[29.2417, "o", "e"] +[29.281, "o", "u"] +[29.321, "o", "_"] +[29.4052, "o", "a"] +[29.447, "o", "i"] +[29.4953, "o", "_"] +[29.5566, "o", "a"] +[29.5965, "o", "c"] +[29.6743, "o", "t"] +[29.8469, "o", " "] +[29.929, "o", "|"] +[30.0082, "o", " "] +[30.0598, "o", "g"] +[30.125, "o", "r"] +[30.1909, "o", "e"] +[30.2435, "o", "p"] +[30.3152, "o", " "] +[30.3924, "o", "-"] +[30.4637, "o", "A"] +[30.5366, "o", "8"] +[30.5898, "o", " "] +[30.6284, "o", "\""] +[30.6883, "o", "p"] +[30.7435, "o", "r"] +[30.7848, "o", "o"] +[30.8245, "o", "d"] +[30.9021, "o", "u"] +[30.9473, "o", "c"] +[30.9886, "o", "e"] +[31.0293, "o", "d"] +[31.0939, "o", " "] +[31.1713, "o", "b"] +[31.2282, "o", "y"] +[31.2966, "o", " "] +[31.368, "o", "e"] +[31.4421, "o", "v"] +[31.5281, "o", "a"] +[31.5911, "o", "l"] +[31.6516, "o", "u"] +[31.7086, "o", "a"] +[31.7847, "o", "t"] +[31.8314, "o", "o"] +[31.8898, "o", "r"] +[31.9389, "o", "s"] +[32.004, "o", "\""] +[32.424, "o", "\r\n"] +[34.484, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] +[34.574, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] +[34.664, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] +[35.454, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] +[35.482, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] +[35.85, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] +[35.878, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] +[35.906, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] +[35.934, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] +[35.962, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] +[35.99, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] +[36.018, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] +[36.046, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] +[36.074, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] +[36.102, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] +[36.13, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] +[36.158, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] +[36.186, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] +[36.214, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] +[36.242, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] +[36.27, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] +[36.298, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] +[42.066, "o", "$ "] +[42.066, "o", "\u2588"] +[42.596, "o", "\b \b"] +[43.126, "o", "\u2588"] +[43.656, "o", "\b \b"] +[44.186, "o", "\u2588"] +[44.716, "o", "\b \b"] +[45.246, "o", "\u2588"] +[45.776, "o", "\b \b"] +[46.306, "o", "\u2588"] +[46.836, "o", "\b \b"] +[47.366, "o", "\u2588"] +[47.896, "o", "\b \b"] diff --git a/docs/demo/aicertify.cast b/docs/demo/aicertify.cast new file mode 100644 index 0000000..84700db --- /dev/null +++ b/docs/demo/aicertify.cast @@ -0,0 +1,534 @@ +{"version": 2, "width": 100, "height": 30, "idle_time_limit": 3.0, "title": "AICertify: what it measures, and what it cannot"} +[0.6, "o", "$ "] +[0.663, "o", "a"] +[0.7371, "o", "i"] +[0.8269, "o", "c"] +[0.8667, "o", "e"] +[0.9498, "o", "r"] +[1.0368, "o", "t"] +[1.1269, "o", "i"] +[1.1942, "o", "f"] +[1.2494, "o", "y"] +[1.4363, "o", " "] +[1.5075, "o", "s"] +[1.5812, "o", "c"] +[1.6367, "o", "o"] +[1.7021, "o", "e"] +[2.1254, "o", "\b \b"] +[2.3135, "o", "r"] +[2.3604, "o", "e"] +[2.4345, "o", "-"] +[2.5228, "o", "c"] +[2.6116, "o", "a"] +[2.7002, "o", "r"] +[2.7515, "o", "d"] +[2.8069, "o", " "] +[2.8613, "o", "b"] +[2.918, "o", "e"] +[2.9962, "o", "r"] +[3.037, "o", "t"] +[3.0848, "o", "-"] +[3.1355, "o", "b"] +[3.1939, "o", "a"] +[3.2771, "o", "s"] +[3.3624, "o", "e"] +[3.4141, "o", "-"] +[3.4592, "o", "u"] +[3.5188, "o", "n"] +[3.588, "o", "c"] +[3.6299, "o", "a"] +[3.6839, "o", "s"] +[3.7495, "o", "e"] +[3.8127, "o", "d"] +[4.2327, "o", "\r\n"] +[6.2927, "o", "\r\nbert-base-uncased\r\n completeness 0.49 (threshold 0.8)\r\n"] +[6.3827, "o", " quality 0.66\r\n BELOW THRESHOLD\r\n\r\n"] +[6.4727, "o", " intended_use 1.00 ####################\r\n training_data 0.67 #############\r\n caveats_recommendations 0.50 ##########\r\n"] +[6.5627, "o", " factors 0.50 ##########\r\n metrics 0.50 ##########\r\n quantitative_analyses 0.50 ##########\r\n"] +[6.6527, "o", " ethical_considerations 0.33 #######\r\n model_details 0.33 ######\r\n evaluation_data 0.00 \r\n"] +[6.7427, "o", "\r\nA model card answers part of what Annex IV asks and then stops.\r\nThis is the documentation-completeness gap, not a compliance verdict.\r\n"] +[6.8327, "o", "\r\nscored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0\r\n"] +[7.7227, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0\u001b[0m\u001b8"] +[7.7507, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] +[7.7787, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] +[7.8067, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] +[8.1747, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] +[8.2027, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] +[8.2307, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] +[8.2587, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] +[8.2867, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] +[8.3147, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] +[8.3427, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] +[8.3707, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] +[8.3987, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] +[8.4267, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] +[8.4547, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] +[8.4827, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] +[8.5107, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] +[8.5387, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] +[8.5667, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] +[14.3347, "o", "$ "] +[14.3728, "o", "#"] +[14.4216, "o", " "] +[14.4751, "o", "s"] +[14.5457, "o", "o"] +[14.6038, "o", " "] +[14.6894, "o", "w"] +[14.7391, "o", "h"] +[14.8281, "o", "a"] +[14.8792, "o", "t"] +[14.9428, "o", " "] +[15.0337, "o", "d"] +[15.1247, "o", "o"] +[15.2054, "o", "e"] +[15.261, "o", "s"] +[15.3275, "o", " "] +[15.3931, "o", "t"] +[15.4326, "o", "h"] +[15.5002, "o", "e"] +[15.5741, "o", " "] +[15.6338, "o", "E"] +[15.6756, "o", "U"] +[15.7291, "o", " "] +[15.8146, "o", "A"] +[15.8902, "o", "I"] +[15.9434, "o", " "] +[16.0148, "o", "A"] +[16.0948, "o", "c"] +[16.146, "o", "t"] +[16.1997, "o", " "] +[16.2682, "o", "a"] +[16.308, "o", "c"] +[16.3465, "o", "t"] +[16.3926, "o", "u"] +[16.4409, "o", "a"] +[16.5198, "o", "l"] +[16.58, "o", "l"] +[16.62, "o", "y"] +[16.7305, "o", " "] +[16.8186, "o", "a"] +[16.8661, "o", "s"] +[16.936, "o", "k"] +[17.004, "o", " "] +[17.0739, "o", "f"] +[17.1166, "o", "o"] +[17.1599, "o", "r"] +[17.2375, "o", "?"] +[17.6575, "o", "\r\n"] +[18.3175, "o", "$ "] +[18.3736, "o", "a"] +[18.4514, "o", "i"] +[18.5393, "o", "c"] +[18.6178, "o", "e"] +[18.6876, "o", "r"] +[18.7719, "o", "t"] +[18.8375, "o", "i"] +[18.8849, "o", "f"] +[18.948, "o", "y"] +[19.12, "o", " "] +[19.2032, "o", "e"] +[19.2881, "o", "x"] +[19.3525, "o", "p"] +[19.4241, "o", "l"] +[19.4642, "o", "a"] +[19.5344, "o", "i"] +[19.6001, "o", "n"] +[19.6664, "o", " "] +[19.727, "o", "e"] +[19.7846, "o", "u"] +[19.8725, "o", "_"] +[19.9425, "o", "a"] +[19.9966, "o", "i"] +[20.0526, "o", "_"] +[20.1064, "o", "s"] +[20.5153, "o", "\b \b"] +[20.7183, "o", "a"] +[20.7907, "o", "c"] +[20.8689, "o", "t"] +[21.0413, "o", " "] +[21.0899, "o", "|"] +[21.169, "o", " "] +[21.2214, "o", "h"] +[21.2811, "o", "e"] +[21.3388, "o", "a"] +[21.3854, "o", "d"] +[21.4432, "o", " "] +[21.5249, "o", "-"] +[21.5912, "o", "1"] +[21.646, "o", "2"] +[22.066, "o", "\r\n"] +[24.126, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] +[24.216, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] +[24.306, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] +[24.396, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] +[25.186, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] +[25.214, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] +[25.242, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] +[28.21, "o", "$ "] +[28.2514, "o", "a"] +[28.3117, "o", "i"] +[28.3522, "o", "c"] +[28.4074, "o", "e"] +[28.4549, "o", "r"] +[28.5189, "o", "t"] +[28.5999, "o", "i"] +[28.6522, "o", "f"] +[28.7288, "o", "y"] +[28.7878, "o", " "] +[28.855, "o", "e"] +[28.9184, "o", "x"] +[28.9987, "o", "p"] +[29.0767, "o", "l"] +[29.1447, "o", "a"] +[29.2237, "o", "i"] +[29.272, "o", "n"] +[29.4492, "o", " "] +[29.5169, "o", "e"] +[29.5878, "o", "u"] +[29.6708, "o", "_"] +[29.7474, "o", "a"] +[29.8036, "o", "i"] +[29.8704, "o", "_"] +[29.9512, "o", "a"] +[30.0082, "o", "c"] +[30.0582, "o", "r"] +[30.4082, "o", "\b \b"] +[30.6003, "o", "t"] +[30.687, "o", " "] +[30.746, "o", "|"] +[30.9139, "o", " "] +[30.9762, "o", "g"] +[31.0533, "o", "r"] +[31.1333, "o", "e"] +[31.1925, "o", "p"] +[31.2527, "o", " "] +[31.3087, "o", "-"] +[31.3607, "o", "A"] +[31.4381, "o", "8"] +[31.6029, "o", " "] +[31.6572, "o", "\""] +[31.7332, "o", "p"] +[31.8124, "o", "r"] +[31.8912, "o", "o"] +[31.9601, "o", "d"] +[32.0083, "o", "u"] +[32.0643, "o", "c"] +[32.1034, "o", "e"] +[32.1912, "o", "d"] +[32.3943, "o", " "] +[32.446, "o", "b"] +[32.5336, "o", "y"] +[32.765, "o", " "] +[32.8458, "o", "e"] +[32.9108, "o", "v"] +[32.9632, "o", "a"] +[33.0423, "o", "l"] +[33.1011, "o", "u"] +[33.1434, "o", "a"] +[33.2204, "o", "t"] +[33.289, "o", "o"] +[33.3575, "o", "r"] +[33.4416, "o", "s"] +[33.5174, "o", "\""] +[33.9374, "o", "\r\n"] +[35.9974, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] +[36.0874, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] +[36.1774, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] +[36.9674, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] +[36.9954, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] +[37.3634, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] +[37.3914, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] +[37.4194, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] +[37.4474, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] +[37.4754, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] +[37.5034, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] +[37.5314, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] +[37.5594, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] +[37.5874, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] +[37.6154, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] +[37.6434, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] +[37.6714, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] +[37.6994, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] +[37.7274, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] +[37.7554, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] +[37.7834, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] +[37.8114, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] +[43.5794, "o", "$ "] +[43.6385, "o", "a"] +[43.6917, "o", "i"] +[43.7727, "o", "c"] +[43.8634, "o", "e"] +[43.9259, "o", "r"] +[43.9854, "o", "t"] +[44.042, "o", "i"] +[44.1211, "o", "f"] +[44.16, "o", "y"] +[44.3086, "o", " "] +[44.3876, "o", "i"] +[44.4676, "o", "n"] +[44.5508, "o", "i"] +[44.6033, "o", "t"] +[44.6617, "o", "-"] +[44.734, "o", "c"] +[44.7916, "o", "o"] +[44.8625, "o", "n"] +[44.9534, "o", "t"] +[45.0147, "o", "r"] +[45.0767, "o", "a"] +[45.1332, "o", "c"] +[45.1976, "o", "t"] +[45.2691, "o", " "] +[45.3283, "o", "0"] +[45.6653, "o", "\b \b"] +[45.8623, "o", "-"] +[45.9455, "o", "-"] +[45.9921, "o", "p"] +[46.0499, "o", "o"] +[46.1131, "o", "l"] +[46.1849, "o", "i"] +[46.2709, "o", "c"] +[46.3439, "o", "y"] +[46.5273, "o", " "] +[46.5701, "o", "e"] +[46.6335, "o", "u"] +[46.7152, "o", "_"] +[46.807, "o", "a"] +[46.8723, "o", "i"] +[46.9585, "o", "_"] +[46.9969, "o", "a"] +[47.0447, "o", "c"] +[47.0982, "o", "t"] +[47.1756, "o", " "] +[47.2153, "o", ">"] +[47.2948, "o", " "] +[47.351, "o", "c"] +[47.4214, "o", "o"] +[47.4799, "o", "n"] +[47.5598, "o", "t"] +[47.6278, "o", "r"] +[47.6705, "o", "a"] +[47.734, "o", "c"] +[47.8129, "o", "t"] +[47.8936, "o", "."] +[47.9715, "o", "j"] +[48.0437, "o", "s"] +[48.1216, "o", "o"] +[48.2074, "o", "n"] +[48.6274, "o", "\r\n"] +[51.5874, "o", "$ "] +[51.6272, "o", "a"] +[51.7191, "o", "i"] +[51.7606, "o", "c"] +[51.8272, "o", "e"] +[51.8975, "o", "r"] +[51.9475, "o", "t"] +[52.0156, "o", "i"] +[52.1, "o", "f"] +[52.1779, "o", "y"] +[52.3785, "o", " "] +[52.4637, "o", "e"] +[52.5155, "o", "v"] +[52.5743, "o", "a"] +[52.6317, "o", "l"] +[52.6783, "o", "u"] +[52.7438, "o", "a"] +[52.8299, "o", "t"] +[52.8931, "o", "e"] +[52.9803, "o", " "] +[53.0393, "o", "-"] +[53.1041, "o", "-"] +[53.1831, "o", "c"] +[53.2314, "o", "o"] +[53.2756, "o", "n"] +[53.346, "o", "t"] +[53.433, "o", "r"] +[53.4931, "o", "a"] +[53.5609, "o", "c"] +[53.6446, "o", "t"] +[53.7168, "o", " "] +[53.7934, "o", "c"] +[53.8616, "o", "o"] +[53.9169, "o", "n"] +[53.9696, "o", "t"] +[54.0341, "o", "r"] +[54.1098, "o", "a"] +[54.1802, "o", "c"] +[54.2676, "o", "t"] +[54.3384, "o", "."] +[54.4258, "o", "j"] +[54.5065, "o", "s"] +[54.563, "o", "o"] +[54.6023, "o", "n"] +[54.6764, "o", " "] +[54.7407, "o", "-"] +[54.7891, "o", "-"] +[54.8412, "o", "p"] +[54.8814, "o", "o"] +[54.9483, "o", "l"] +[54.9985, "o", "i"] +[55.0459, "o", "c"] +[55.1294, "o", "y"] +[55.3245, "o", " "] +[55.3683, "o", "e"] +[55.4425, "o", "u"] +[55.4895, "o", "_"] +[55.5737, "o", "a"] +[55.627, "o", "i"] +[55.7134, "o", "_"] +[55.7583, "o", "a"] +[55.8154, "o", "c"] +[55.8796, "o", "t"] +[56.2996, "o", "\r\n"] +[58.3596, "o", "Evaluation Summary:\r\nContract ID: b7e60f21-6717-4ebd-b83f-dd6927dd5c52\r\nApplication: your-application\r\n"] +[58.4496, "o", "Report: ./reports/folder_report_your-application_2026-08-30_165312.md\r\nOPA Evaluation: Successful\r\n"] +[59.3396, "o", "$ "] +[59.4066, "o", "h"] +[59.4879, "o", "e"] +[59.5623, "o", "a"] +[59.652, "o", "d"] +[59.73, "o", " "] +[59.7905, "o", "-"] +[59.8812, "o", "8"] +[59.962, "o", " "] +[60.0213, "o", "e"] +[60.3724, "o", "\b \b"] +[60.5701, "o", "r"] +[60.6452, "o", "e"] +[60.7252, "o", "p"] +[60.7674, "o", "o"] +[60.8375, "o", "r"] +[60.8975, "o", "t"] +[60.9699, "o", "s"] +[61.0196, "o", "/"] +[61.08, "o", "*"] +[61.138, "o", "."] +[61.1952, "o", "m"] +[61.287, "o", "d"] +[61.707, "o", "\r\n"] +[62.167, "o", "# Evaluation Report\r\n\r\n**Analysis Date:** 2026-08-30 11:23:12\r\n"] +[62.257, "o", "\r\n## Evaluation Summary\r\n- **Total Policies:** 29\r\n"] +[62.347, "o", "- **Green Count:** 0\r\n- **Red Count:** 29\r\n"] +[63.137, "o", "\u001b7\u001b[1A\r\u001b[17C\u001b[43;30m2\u001b[0m\u001b8"] +[63.165, "o", "\u001b7\u001b[1A\r\u001b[17C\u001b[43;30m29\u001b[0m\u001b8"] +[63.533, "o", "\u001b7\u001b[2A\r\u001b[19C\u001b[43;30m0\u001b[0m\u001b8"] +[69.301, "o", "$ "] +[69.3882, "o", "#"] +[69.4743, "o", " "] +[69.5451, "o", "w"] +[69.6227, "o", "h"] +[69.7075, "o", "i"] +[69.7818, "o", "c"] +[69.828, "o", "h"] +[69.9057, "o", " "] +[69.9455, "o", "m"] +[69.9909, "o", "e"] +[70.0551, "o", "t"] +[70.0977, "o", "r"] +[70.1655, "o", "i"] +[70.2543, "o", "c"] +[70.2991, "o", "s"] +[70.4012, "o", " "] +[70.4616, "o", "c"] +[70.5501, "o", "a"] +[70.6038, "o", "n"] +[70.6874, "o", " "] +[70.7728, "o", "a"] +[70.8183, "o", "n"] +[70.9047, "o", " "] +[70.9633, "o", "e"] +[71.0446, "o", "v"] +[71.1189, "o", "a"] +[71.2031, "o", "l"] +[71.2921, "o", "u"] +[71.3711, "o", "a"] +[71.4169, "o", "t"] +[71.4596, "o", "o"] +[71.5339, "o", "r"] +[71.7245, "o", " "] +[71.7665, "o", "s"] +[71.8055, "o", "u"] +[71.8918, "o", "p"] +[71.9756, "o", "p"] +[72.03, "o", "l"] +[72.0698, "o", "y"] +[72.1141, "o", " "] +[72.1811, "o", "t"] +[72.2331, "o", "o"] +[72.3002, "o", "d"] +[72.3897, "o", "a"] +[72.4633, "o", "y"] +[72.5286, "o", "?"] +[72.9486, "o", "\r\n"] +[73.6086, "o", "$ "] +[73.6944, "o", "p"] +[73.7683, "o", "y"] +[73.8566, "o", "t"] +[73.9233, "o", "h"] +[74.0105, "o", "o"] +[74.0865, "o", "n"] +[74.2597, "o", " "] +[74.3029, "o", "s"] +[74.3827, "o", "c"] +[74.4661, "o", "r"] +[74.5439, "o", "i"] +[74.6249, "o", "p"] +[74.7139, "o", "t"] +[74.797, "o", "s"] +[74.8734, "o", "/"] +[74.9156, "o", "m"] +[74.9747, "o", "e"] +[75.0364, "o", "t"] +[75.1187, "o", "r"] +[75.1817, "o", "i"] +[75.261, "o", "c"] +[75.3511, "o", "_"] +[75.3951, "o", "g"] +[75.4514, "o", "a"] +[75.5039, "o", "p"] +[75.594, "o", "_"] +[75.6727, "o", "r"] +[75.7639, "o", "e"] +[75.8538, "o", "p"] +[75.9081, "o", "o"] +[75.9502, "o", "r"] +[76.0355, "o", "t"] +[76.0967, "o", "."] +[76.1828, "o", "p"] +[76.2418, "o", "y"] +[76.3131, "o", " "] +[76.3935, "o", "|"] +[76.5115, "o", " "] +[76.5943, "o", "t"] +[76.665, "o", "a"] +[76.7531, "o", "i"] +[76.8212, "o", "l"] +[76.8663, "o", " "] +[76.9249, "o", "-"] +[76.977, "o", "1"] +[77.0476, "o", "2"] +[77.4676, "o", "\r\n"] +[79.5276, "o", " calc metrics.model_card.completeness computed by global.v1.documentation.model_card_score, no evaluator needed\r\n GAP metrics.model_card.compliance_level no evaluator declares this\r\n calc metrics.model_card.quality computed by global.v1.documentation.model_card_score, no evaluator needed\r\n"] +[79.6176, "o", " calc metrics.model_card.section_scores computed by global.v1.documentation.model_card_score, no evaluator needed\r\n ok metrics.social_scoring.combined_score SocialScoringEvaluator\r\n ok metrics.social_scoring.detrimental_treatment SocialScoringEvaluator\r\n"] +[79.7076, "o", " ok metrics.social_scoring.score SocialScoringEvaluator\r\n ok metrics.toxicity.max_toxicity ContentSafetyEvaluator\r\n\r\n"] +[79.7976, "o", "TOTAL: 22 of 27 measured metrics can be supplied today.\r\nThe gaps are what you would need to write an evaluator for, or\r\nsupply by hand. See docs on writing one.\r\n"] +[80.6876, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m2\u001b[0m\u001b8"] +[80.7156, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22\u001b[0m\u001b8"] +[80.7436, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 \u001b[0m\u001b8"] +[80.7716, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 o\u001b[0m\u001b8"] +[80.7996, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of\u001b[0m\u001b8"] +[80.8276, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of \u001b[0m\u001b8"] +[80.8556, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of 2\u001b[0m\u001b8"] +[80.8836, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of 27\u001b[0m\u001b8"] +[86.6516, "o", "$ "] +[86.6516, "o", "\u2588"] +[87.1816, "o", "\b \b"] +[87.7116, "o", "\u2588"] +[88.2416, "o", "\b \b"] +[88.7716, "o", "\u2588"] +[89.3016, "o", "\b \b"] +[89.8316, "o", "\u2588"] +[90.3616, "o", "\b \b"] +[90.8916, "o", "\u2588"] +[91.4216, "o", "\b \b"] +[91.9516, "o", "\u2588"] +[92.4816, "o", "\b \b"] diff --git a/scripts/record-demo.py b/scripts/record-demo.py new file mode 100644 index 0000000..9bbfa6d --- /dev/null +++ b/scripts/record-demo.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +"""Compose the AICertify demo asciicast: real output, typed at a human pace. + +Every command here is actually run and its output captured, so nothing on the +screen is written by hand. What is synthesised is the *timing*. The four +commands take about three minutes between them, most of it downloading model +weights and waiting on OPA, and a cast at real speed is unwatchable. A `script` +capture would also have no typing in it at all, and output that materialises +instantly reads as a screenshot rather than a session. + +Being explicit about which half is real matters. The scores, the field counts, +the verdicts and the report are what the commands printed. The rhythm is +invented, the way any screencast's rhythm is invented by the person recording. + +Four acts, because AICertify does a different job from GOPAL and needs the +room. GOPAL decides things, so its demo is one beat: swap the model, the build +fails. AICertify gathers things, and the interesting thing about it is how +honest it is about the part it cannot gather. + + 1. score-card a real Hugging Face card, scored, in one screen + 2. explain 150 fields you must declare, 14 an evaluator computes + 3. init/eval scaffold a contract, run it, get 0 of 29 + 4. gap report 22 of 27 metrics supplied, and the five that are not + +Act 3 ends on nothing passing. That is the point rather than an embarrassment: +the scaffold asserts none of the 150 declarations, so none of the obligations +that turn on them can be met. It is the same argument act 2 makes, arriving as +a verdict instead of a number. + +Usage: + record-demo.py [short.cast] + +The short cast is acts 1 and 2 only, for the README, where a 90-second loop is +too long and JavaScript does not run. +""" + +from __future__ import annotations + +import json +import random +import math +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent +PY = str(REPO / ".venv" / "bin" / "python") +CLI = [PY, "-m", "aicertify.cli"] + +# Deterministic jitter. The keystrokes look uneven, and they are the same +# uneven every time, so regenerating does not churn the file in review. +RNG = random.Random(20260830) + +CPS_MIN, CPS_MAX = 0.038, 0.092 # seconds per keystroke +REACH_FOR_ENTER = 0.42 # hand leaves the letters, finds the return key +ENTER_SETTLES = 0.16 # and the press registers before anything happens +FIRST_BYTE = 0.30 # a command does not answer instantly +THINKING = 1.9 # a command that clearly did some work +READ_SHORT = 2.6 +READ_LONG = 5.4 # long enough to read a verdict and understand it +BLINK = 0.53 +BLINKS = 6 +CURSOR = "█" + +SWEEP = 0.028 # per character, the pace of a pen moving +HL_ON = "\x1b[43;30m" +HL_OFF = "\x1b[0m" +SAVE, RESTORE = "\x1b7", "\x1b8" + +TYPO_CHANCE = 0.55 # per command, not per character +NOTICE_TYPO = (0.22, 0.46) +AFTER_FIX = (0.08, 0.19) + +NEIGHBOURS = { + "a": "sq", + "b": "vn", + "c": "xv", + "d": "sf", + "e": "wr", + "f": "dg", + "g": "fh", + "h": "gj", + "i": "uo", + "j": "hk", + "k": "jl", + "l": "k", + "m": "n", + "n": "bm", + "o": "ip", + "p": "o", + "q": "wa", + "r": "et", + "s": "ad", + "t": "ry", + "u": "yi", + "v": "cb", + "w": "qe", + "x": "zc", + "y": "tu", + "z": "x", + ".": "/", + "/": ".", + "-": "0", + "_": "-", +} + +WIDTH, HEIGHT = 100, 30 + + +class Cast: + def __init__(self) -> None: + self.t = 0.0 + self.events: list[list] = [] + + def wait(self, seconds: float) -> None: + self.t += seconds + + def out(self, text: str) -> None: + self.events.append([round(self.t, 4), "o", text]) + + def type(self, text: str, typo_at: int | None = None) -> None: + """One keystroke at a time, with the pauses and the mistakes. + + A typo is the neighbouring key, noticed a moment later, backspaced and + retyped. It is the single thing that separates a recording of somebody + working from text replayed at a plausible speed. + """ + for i, ch in enumerate(text): + if i == typo_at: + wrong = NEIGHBOURS.get(ch.lower()) + if wrong: + self.wait(RNG.uniform(CPS_MIN, CPS_MAX)) + self.out(RNG.choice(wrong)) + self.wait(RNG.uniform(*NOTICE_TYPO)) + self.out("\b \b") + self.wait(RNG.uniform(*AFTER_FIX)) + self.wait(RNG.uniform(CPS_MIN, CPS_MAX)) + if ch == " " and RNG.random() < 0.35: + self.wait(RNG.uniform(0.05, 0.16)) + self.out(ch) + + def command(self, text: str) -> None: + self.out("$ ") + typo_at = None + if len(text) > 14 and RNG.random() < TYPO_CHANCE: + typo_at = RNG.randrange(6, len(text) - 2) + self.type(text, typo_at) + # Pressing return is two beats: reaching for the key, and the press + # landing. Without them a command line ends and output begins in the + # same instant, which nothing on a real keyboard does. + self.wait(REACH_FOR_ENTER) + self.out("\r\n") + self.wait(ENTER_SETTLES) + + def block(self, text: str, first_byte: float = FIRST_BYTE) -> None: + """Output arriving in a few pieces, as a real command does.""" + self.wait(first_byte) + lines = text.rstrip("\n").split("\n") + for i in range(0, len(lines), 3): + self.out("\r\n".join(lines[i : i + 3]) + "\r\n") + self.wait(0.09) + + def blink(self, times: int = BLINKS) -> None: + self.out("$ ") + for _ in range(times): + self.out(CURSOR) + self.wait(BLINK) + self.out("\b \b") + self.wait(BLINK) + + def highlight(self, rows_up: int, col: int, text: str) -> None: + """Draw a highlighter left to right over text already on screen.""" + # CSI 0 C moves the cursor forward one column, not zero: ANSI reads a + # zero parameter as one. Emitting it for a phrase at column 0 draws the + # highlight one place right and leaves the original first letter behind. + forward = f"\x1b[{col}C" if col > 0 else "" + for i in range(1, len(text) + 1): + self.out( + f"{SAVE}\x1b[{rows_up}A\r{forward}" + f"{HL_ON}{text[:i]}{HL_OFF}{RESTORE}" + ) + self.wait(SWEEP) + + def sweep(self, block: str, pairs: list[tuple[str, str]]) -> None: + """Highlight each phrase in a block of output just written. + + The cursor is moved up by *screen rows*, not by lines. A line longer + than the terminal is wrapped by the emulator and occupies several rows, + so counting lines puts the highlighter above where the text actually + is. Two 130-character warnings at the end of one command were enough to + draw `BELOW THRESHOLD` across an unrelated score three rows below it. + """ + lines = block.rstrip("\n").split("\n") + + def rows(line: str) -> int: + return max(1, math.ceil(len(line) / WIDTH)) + + for needle, phrase in pairs: + for idx, line in enumerate(lines): + if needle in line and phrase in line: + up = sum(rows(rest) for rest in lines[idx:]) + self.highlight(up, line.index(phrase), phrase) + self.wait(0.34) + break + + def write(self, dst: Path, title: str) -> None: + header = { + "version": 2, + "width": WIDTH, + "height": HEIGHT, + "idle_time_limit": 3.0, + "title": title, + } + with dst.open("w", encoding="utf-8") as fh: + fh.write(json.dumps(header) + "\n") + for event in self.events: + fh.write(json.dumps(event) + "\n") + print(f"Wrote {dst} ({len(self.events)} events, {self.t:.1f}s)") + + +def run(args: list[str], cwd: Path | None = None, stderr: bool = True) -> str: + done = subprocess.run( + args, cwd=cwd or REPO, capture_output=True, text=True, timeout=900 + ) + out = done.stdout or "" + # stderr is dropped where it carries only environment noise, such as the + # Hugging Face hub advising you to set a token. It is real, but it is not + # the command answering the question, and it would leave the act ending on + # a warning rather than on the verdict. + return out + (done.stderr or "") if stderr else out + + +def shell(cmd: str, cwd: Path | None = None) -> str: + done = subprocess.run( + cmd, cwd=cwd or REPO, shell=True, capture_output=True, text=True, timeout=900 + ) + return (done.stdout or "") + (done.stderr or "") + + +def require(condition: bool, message: str) -> None: + if not condition: + print(f"refusing to record: {message}", file=sys.stderr) + raise SystemExit(1) + + +def main() -> int: + if len(sys.argv) not in (2, 3): + print(__doc__.strip(), file=sys.stderr) + return 2 + full_dst = Path(sys.argv[1]) + short_dst = Path(sys.argv[2]) if len(sys.argv) == 3 else None + + work = Path(tempfile.mkdtemp(prefix="aicertify-demo-")) + try: + print("running the commands (this takes a few minutes)...", file=sys.stderr) + + card = run(CLI + ["score-card", "bert-base-uncased"], stderr=False) + declare = shell( + f"{PY} -m aicertify.cli explain eu_ai_act 2>/dev/null | head -12" + ) + measure = shell( + f"{PY} -m aicertify.cli explain eu_ai_act 2>/dev/null" + ' | grep -A8 "produced by evaluators"' + ) + + contract = work / "contract.json" + shell( + f"{PY} -m aicertify.cli init-contract --policy eu_ai_act " + f"2>/dev/null > {contract}" + ) + require( + contract.exists() and contract.stat().st_size > 1000, + "init-contract produced no usable contract", + ) + # A declarations-only contract. The scaffold ships one placeholder + # interaction, and evaluating it pulls DeepEval into an LLM call that + # needs a provider key. The point of this act is the policy verdict, + # not the toxicity classifier, and act 4 covers the evaluators. + doc = json.loads(contract.read_text()) + doc["interactions"] = [] + contract.write_text(json.dumps(doc, indent=2)) + + evaluate = shell( + f"{PY} -m aicertify.cli evaluate --contract contract.json " + f"--policy eu_ai_act --report-format markdown --output-dir ./reports " + "2>&1 | tail -5", + cwd=work, + ) + summary = shell("head -8 reports/*.md", cwd=work) + gaps = shell(f"{PY} scripts/metric_gap_report.py 2>/dev/null | tail -12") + + # Guards. Each of these is a fact the cast puts on screen, so if the + # tool stops saying it the recording is a lie and must not be written. + require( + "0.49" in card and "BELOW THRESHOLD" in card, + "bert-base-uncased no longer scores 0.49 below the threshold", + ) + require( + "Fields you must declare (150)" in declare, + "explain no longer reports 150 declared fields; it disagrees " + "with the published figure and one of them is wrong", + ) + require( + "Fields produced by evaluators (14)" in measure, + "explain no longer reports 14 measured fields", + ) + require( + "defeats the point" in measure, + "the line explaining why you must not hand-write metrics is gone", + ) + require( + "Red Count:** 29" in summary and "Green Count:** 0" in summary, + "an empty contract no longer fails all 29 policies, so act 3 " + "has lost its point", + ) + require( + "22 of 27" in gaps, + "the metric gap report no longer says 22 of 27; act 4's " + "contribution hook needs the real number", + ) + + acts = build(card, declare, measure, evaluate, summary, gaps) + acts.write(full_dst, "AICertify: what it measures, and what it cannot") + + if short_dst: + short = build_short(card, declare, measure) + short.write(short_dst, "AICertify: 150 declared, 14 measured") + return 0 + finally: + shutil.rmtree(work, ignore_errors=True) + + +def act_one(cast: Cast, card: str) -> None: + """A famous model, scored, with no setup at all.""" + cast.wait(0.6) + cast.command("aicertify score-card bert-base-uncased") + cast.block(card, first_byte=THINKING) + cast.wait(0.8) + cast.sweep( + card, + [ + ("completeness", "0.49"), + ("BELOW THRESHOLD", "BELOW THRESHOLD"), + ], + ) + cast.wait(READ_LONG) + + +def act_two(cast: Cast, declare: str, measure: str) -> None: + """The split that the whole product turns on.""" + cast.command("# so what does the EU AI Act actually ask for?") + cast.wait(0.5) + cast.command("aicertify explain eu_ai_act | head -12") + cast.block(declare, first_byte=THINKING) + cast.wait(0.7) + cast.sweep(declare, [("Fields you must declare", "150")]) + cast.wait(READ_SHORT) + + cast.command('aicertify explain eu_ai_act | grep -A8 "produced by evaluators"') + cast.block(measure, first_byte=THINKING) + cast.wait(0.7) + cast.sweep( + measure, + [ + ("produced by evaluators", "14"), + ("defeats the point", "defeats the point"), + ], + ) + cast.wait(READ_LONG) + + +def build_short(card: str, declare: str, measure: str) -> Cast: + cast = Cast() + act_one(cast, card) + act_two(cast, declare, measure) + cast.blink() + return cast + + +def build(card, declare, measure, evaluate, summary, gaps) -> Cast: + cast = Cast() + act_one(cast, card) + act_two(cast, declare, measure) + + # Act 3: scaffold, run, and find out you have answered nothing. + cast.command("aicertify init-contract --policy eu_ai_act > contract.json") + cast.wait(THINKING + 0.9) + cast.command("aicertify evaluate --contract contract.json --policy eu_ai_act") + cast.block(evaluate, first_byte=THINKING) + cast.wait(0.8) + + cast.command("head -8 reports/*.md") + cast.block(summary) + cast.wait(0.7) + cast.sweep(summary, [("Red Count", "29"), ("Green Count", "0")]) + cast.wait(READ_LONG) + + # Act 4: the tool enumerating its own gaps, which is a contributor's + # to-do list rather than a caveat. + cast.command("# which metrics can an evaluator supply today?") + cast.wait(0.5) + cast.command("python scripts/metric_gap_report.py | tail -12") + cast.block(gaps, first_byte=THINKING) + cast.wait(0.8) + cast.sweep(gaps, [("TOTAL", "22 of 27")]) + cast.wait(READ_LONG) + cast.blink() + return cast + + +if __name__ == "__main__": + raise SystemExit(main())