From 27891404c07b2b9732df37cbe9b194acb8d5a92d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:43:32 +0400 Subject: [PATCH 1/3] Prove the gateway and executor Pyright rules as enforced rules Their proofs compared a recorded report with a fixture baseline. With no errors left, each proof now checks what an enforced rule allows: the bad fixture is the package's real report from before its fixes (main at ceb30e17), which fails with every error new, and the good fixture is a clean report recorded from the fixed package. Co-Authored-By: Claude Opus 5.5 --- .../bad/baseline.txt | 5 -- .../good/baseline.txt | 4 -- .../good/pyright.json | 55 +------------------ .../bad/baseline.txt | 5 -- .../good/baseline.txt | 4 -- .../good/pyright.json | 53 +----------------- scripts/rules/tests/test_pyright.py | 32 ++++++----- 7 files changed, 22 insertions(+), 136 deletions(-) delete mode 100644 rules/fixtures/sim.quality.pyright-executor/bad/baseline.txt delete mode 100644 rules/fixtures/sim.quality.pyright-executor/good/baseline.txt delete mode 100644 rules/fixtures/sim.quality.pyright-gateway/bad/baseline.txt delete mode 100644 rules/fixtures/sim.quality.pyright-gateway/good/baseline.txt diff --git a/rules/fixtures/sim.quality.pyright-executor/bad/baseline.txt b/rules/fixtures/sim.quality.pyright-executor/bad/baseline.txt deleted file mode 100644 index 099021df0..000000000 --- a/rules/fixtures/sim.quality.pyright-executor/bad/baseline.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Proof fixture: the baseline omits the "Secret" error in pyright.json, and -# still lists an "Image" error that no longer occurs. -projects/policyengine-simulation-executor/src/modal/app.py :: reportAttributeAccessIssue :: "App" is not a known attribute of module "modal" -projects/policyengine-simulation-executor/src/modal/app.py :: reportAttributeAccessIssue :: "Image" is not a known attribute of module "modal" -projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py :: reportPossiblyUnboundVariable :: "metadata" is possibly unbound diff --git a/rules/fixtures/sim.quality.pyright-executor/good/baseline.txt b/rules/fixtures/sim.quality.pyright-executor/good/baseline.txt deleted file mode 100644 index 541795061..000000000 --- a/rules/fixtures/sim.quality.pyright-executor/good/baseline.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Proof fixture: the baseline lists exactly the errors in pyright.json. -projects/policyengine-simulation-executor/src/modal/app.py :: reportAttributeAccessIssue :: "App" is not a known attribute of module "modal" -projects/policyengine-simulation-executor/src/modal/app.py :: reportAttributeAccessIssue :: "Secret" is not a known attribute of module "modal" -projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py :: reportPossiblyUnboundVariable :: "metadata" is possibly unbound diff --git a/rules/fixtures/sim.quality.pyright-executor/good/pyright.json b/rules/fixtures/sim.quality.pyright-executor/good/pyright.json index 668fe4ddb..4d419d935 100644 --- a/rules/fixtures/sim.quality.pyright-executor/good/pyright.json +++ b/rules/fixtures/sim.quality.pyright-executor/good/pyright.json @@ -1,58 +1,9 @@ { "version": "1.1.414", - "generalDiagnostics": [ - { - "file": "/repository/projects/policyengine-simulation-executor/src/modal/app.py", - "severity": "error", - "message": "\"App\" is not a known attribute of module \"modal\"", - "range": { - "start": { - "line": 118, - "character": 12 - }, - "end": { - "line": 118, - "character": 15 - } - }, - "rule": "reportAttributeAccessIssue" - }, - { - "file": "/repository/projects/policyengine-simulation-executor/src/modal/app.py", - "severity": "error", - "message": "\"Secret\" is not a known attribute of module \"modal\"", - "range": { - "start": { - "line": 122, - "character": 19 - }, - "end": { - "line": 122, - "character": 25 - } - }, - "rule": "reportAttributeAccessIssue" - }, - { - "file": "/repository/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py", - "severity": "error", - "message": "\"metadata\" is possibly unbound", - "range": { - "start": { - "line": 823, - "character": 15 - }, - "end": { - "line": 823, - "character": 23 - } - }, - "rule": "reportPossiblyUnboundVariable" - } - ], + "generalDiagnostics": [], "summary": { - "filesAnalyzed": 90, - "errorCount": 3, + "filesAnalyzed": 91, + "errorCount": 0, "warningCount": 0, "informationCount": 0 } diff --git a/rules/fixtures/sim.quality.pyright-gateway/bad/baseline.txt b/rules/fixtures/sim.quality.pyright-gateway/bad/baseline.txt deleted file mode 100644 index 673fa595f..000000000 --- a/rules/fixtures/sim.quality.pyright-gateway/bad/baseline.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Proof fixture: pyright.json has two "route" errors and the baseline lists -# one, and the baseline still lists a "bundle" error that no longer occurs. -projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py :: reportPossiblyUnboundVariable :: "route" is possibly unbound -projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py :: reportPossiblyUnboundVariable :: "payload" is possibly unbound -projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py :: reportPossiblyUnboundVariable :: "bundle" is possibly unbound diff --git a/rules/fixtures/sim.quality.pyright-gateway/good/baseline.txt b/rules/fixtures/sim.quality.pyright-gateway/good/baseline.txt deleted file mode 100644 index 3a7b94654..000000000 --- a/rules/fixtures/sim.quality.pyright-gateway/good/baseline.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Proof fixture: the baseline lists exactly the errors in pyright.json. -projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py :: reportPossiblyUnboundVariable :: "route" is possibly unbound -projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py :: reportPossiblyUnboundVariable :: "route" is possibly unbound -projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py :: reportPossiblyUnboundVariable :: "payload" is possibly unbound diff --git a/rules/fixtures/sim.quality.pyright-gateway/good/pyright.json b/rules/fixtures/sim.quality.pyright-gateway/good/pyright.json index a902297dd..cd2a79ab8 100644 --- a/rules/fixtures/sim.quality.pyright-gateway/good/pyright.json +++ b/rules/fixtures/sim.quality.pyright-gateway/good/pyright.json @@ -1,58 +1,9 @@ { "version": "1.1.414", - "generalDiagnostics": [ - { - "file": "/repository/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py", - "severity": "error", - "message": "\"route\" is possibly unbound", - "range": { - "start": { - "line": 922, - "character": 26 - }, - "end": { - "line": 922, - "character": 31 - } - }, - "rule": "reportPossiblyUnboundVariable" - }, - { - "file": "/repository/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py", - "severity": "error", - "message": "\"route\" is possibly unbound", - "range": { - "start": { - "line": 923, - "character": 25 - }, - "end": { - "line": 923, - "character": 30 - } - }, - "rule": "reportPossiblyUnboundVariable" - }, - { - "file": "/repository/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py", - "severity": "error", - "message": "\"payload\" is possibly unbound", - "range": { - "start": { - "line": 934, - "character": 8 - }, - "end": { - "line": 934, - "character": 15 - } - }, - "rule": "reportPossiblyUnboundVariable" - } - ], + "generalDiagnostics": [], "summary": { "filesAnalyzed": 9, - "errorCount": 3, + "errorCount": 0, "warningCount": 0, "informationCount": 0 } diff --git a/scripts/rules/tests/test_pyright.py b/scripts/rules/tests/test_pyright.py index 378de485d..7e319b4c7 100644 --- a/scripts/rules/tests/test_pyright.py +++ b/scripts/rules/tests/test_pyright.py @@ -190,35 +190,37 @@ def test_rule_proof_pyright_libraries_and_entry(self) -> None: def test_rule_proof_pyright_gateway(self) -> None: rule_id = "sim.quality.pyright-gateway" - # The baseline lists one of two "route" errors, and an error that no - # longer occurs. + # The gateway's real errors before its fix: variables assigned inside + # a span, which Pyright took to swallow exceptions. bad = proof_comparison(rule_id, "bad") self.assertEqual( [(error.line, error.summary) for error in bad.new_errors], - [(924, '"route" is possibly unbound')], - ) - self.assertEqual( - bad.stale_entries, - (f'{GATEWAY_ENDPOINTS} :: {UNBOUND} :: "bundle" is possibly unbound',), + [ + (923, '"route" is possibly unbound'), + (924, '"route" is possibly unbound'), + (935, '"payload" is possibly unbound'), + ], ) + self.assertEqual(bad.stale_entries, ()) self.assertTrue(proof_comparison(rule_id, "good").passed) def test_rule_proof_pyright_executor(self) -> None: rule_id = "sim.quality.pyright-executor" + # The executor's real errors before its fixes: modal attributes + # unknown while src/modal shadowed the library, and a variable + # assigned inside a span. bad = proof_comparison(rule_id, "bad") self.assertEqual( [(error.line, error.summary) for error in bad.new_errors], - [(123, '"Secret" is not a known attribute of module "modal"')], - ) - self.assertEqual( - bad.stale_entries, - ( - f"{EXECUTOR_APP} :: reportAttributeAccessIssue :: " - '"Image" is not a known attribute of module "modal"', - ), + [ + (119, '"App" is not a known attribute of module "modal"'), + (123, '"Secret" is not a known attribute of module "modal"'), + (824, '"metadata" is possibly unbound'), + ], ) + self.assertEqual(bad.stale_entries, ()) self.assertTrue(proof_comparison(rule_id, "good").passed) def test_rule_proof_pyright_apis_integ(self) -> None: From 7440c0b0a05f166df76cb0268addc956f046460e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:43:41 +0400 Subject: [PATCH 2/3] Enforce Pyright for the gateway and executor, and drop their baselines Both packages have no Pyright errors, so sim.quality.pyright-gateway and sim.quality.pyright-executor move from ratcheted to enforced, and their empty baselines go. apis-integ stays ratcheted at 4. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/type-checking.md | 12 ++++++------ docs/generated/rules.md | 12 +++++------- rules/baselines/pyright-executor.txt | 9 --------- rules/baselines/pyright-gateway.txt | 9 --------- rules/registry.toml | 18 ++++++------------ 5 files changed, 17 insertions(+), 43 deletions(-) delete mode 100644 rules/baselines/pyright-executor.txt delete mode 100644 rules/baselines/pyright-gateway.txt diff --git a/docs/engineering/skills/type-checking.md b/docs/engineering/skills/type-checking.md index c4f16d4d8..5d2b83cf6 100644 --- a/docs/engineering/skills/type-checking.md +++ b/docs/engineering/skills/type-checking.md @@ -9,12 +9,12 @@ package. Every uv package is type-checked with Pyright on every pull request, by the `pyright` job in `.github/workflows/pr.yml`: -- `sim.quality.pyright-libraries-and-entry`: the contract, observability, - Stage 12 persistence and fastapi libraries and the simulation entry have no - Pyright errors. -- `sim.quality.pyright-gateway`, `sim.quality.pyright-executor` and - `sim.quality.pyright-apis-integ`: these packages may have only the errors - their baselines list (see [Baselines](#baselines)). +- `sim.quality.pyright-libraries-and-entry`, `sim.quality.pyright-gateway` and + `sim.quality.pyright-executor`: the contract, observability, Stage 12 + persistence and fastapi libraries, the simulation entry, the gateway and the + executor have no Pyright errors. +- `sim.quality.pyright-apis-integ`: the integration tests may have only the + errors their baseline lists (see [Baselines](#baselines)). - `sim.quality.every-package-type-checked`: a Pyright rule lists every uv package under `libs/` and `projects/`. When you add a package, give it a `[tool.pyright] include`, add it to a Pyright rule's `enforcer.projects`, and diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 2d205e006..ea6b98965 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,7 +8,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 49 | 5 | 0 | 15 | 69 | +| mechanical | 51 | 3 | 0 | 15 | 69 | | metric | 0 | 0 | 0 | 0 | 0 | | judgment | 0 | 0 | 0 | 0 | 0 | @@ -68,8 +68,6 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf - `sim.ci.multiline-run-is-script`: 13 remaining in `rules/baselines/ci-multiline-run.txt` - `sim.microdata.weight-stripping-confined`: 1 remaining in `rules/baselines/sim-microdata-weight-stripping.txt` - `sim.quality.pyright-apis-integ`: 4 remaining in `rules/baselines/pyright-apis-integ.txt` -- `sim.quality.pyright-executor`: 0 remaining in `rules/baselines/pyright-executor.txt` -- `sim.quality.pyright-gateway`: 0 remaining in `rules/baselines/pyright-gateway.txt` ### Exceptions and review dates @@ -292,8 +290,8 @@ flowchart LR | `sim.quality.every-package-type-checked` | Every uv package under libs/ and projects/ (a directory with a pyproject.toml) is listed by a Pyright rule, so the pyright CI job type-checks it. | mechanical | enforced | yes | quality guard `pyright-coverage` | `pr.yml:quality-guards` | yes | | `sim.quality.every-project-tested` | Every project and library with unit tests runs them in the PR test matrix. | mechanical | doc-only | no | nothing | — | — | | `sim.quality.pyright-apis-integ` | The generated-client integration-test project, apis-integ, has no Pyright errors beyond those its baseline lists, and the baseline lists only errors that still occur, so it only shrinks; checked with the pinned Pyright version in the project's own locked environment. | mechanical | ratcheted | yes | pyright in `projects/policyengine-apis-integ` | `pr.yml:pyright` | yes | -| `sim.quality.pyright-executor` | The simulation executor has no Pyright errors beyond those its baseline lists, and the baseline lists only errors that still occur, so it only shrinks; checked with the pinned Pyright version in the executor's own locked environment. | mechanical | ratcheted | yes | pyright in `projects/policyengine-simulation-executor` | `pr.yml:pyright` | yes | -| `sim.quality.pyright-gateway` | The simulation gateway has no Pyright errors beyond those its baseline lists, and the baseline lists only errors that still occur, so it only shrinks; checked with the pinned Pyright version in the gateway's own locked environment. | mechanical | ratcheted | yes | pyright in `projects/policyengine-simulation-gateway` | `pr.yml:pyright` | yes | +| `sim.quality.pyright-executor` | The simulation executor has no Pyright errors, checked with the pinned Pyright version in the executor's own locked environment. | mechanical | enforced | yes | pyright in `projects/policyengine-simulation-executor` | `pr.yml:pyright` | yes | +| `sim.quality.pyright-gateway` | The simulation gateway has no Pyright errors, checked with the pinned Pyright version in the gateway's own locked environment. | mechanical | enforced | yes | pyright in `projects/policyengine-simulation-gateway` | `pr.yml:pyright` | yes | | `sim.quality.pyright-libraries-and-entry` | The contract, observability, Stage 12 persistence and fastapi libraries and the simulation entry have no Pyright errors, checked with the pinned Pyright version in each package's own locked environment. | mechanical | enforced | yes | pyright in `libs/policyengine-simulation-contract`, `libs/policyengine-simulation-observability`, `libs/policyengine-stage12-persistence`, `libs/policyengine-fastapi`, `projects/policyengine-simulation-entry` | `pr.yml:pyright` | yes | | `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — | @@ -321,9 +319,9 @@ flowchart LR r_sim_quality_pyright_apis_integ("sim.quality.pyright-apis-integ (ratcheted)") e_pyright_in_projects_policyengine_apis_integ["pyright in projects/policyengine-apis-integ"] j_CI_pr_yml_pyright["CI pr.yml:pyright"] - r_sim_quality_pyright_executor("sim.quality.pyright-executor (ratcheted)") + r_sim_quality_pyright_executor("sim.quality.pyright-executor (enforced)") e_pyright_in_projects_policyengine_simulation_executor["pyright in projects/policyengine-simulation-executor"] - r_sim_quality_pyright_gateway("sim.quality.pyright-gateway (ratcheted)") + r_sim_quality_pyright_gateway("sim.quality.pyright-gateway (enforced)") e_pyright_in_projects_policyengine_simulation_gateway["pyright in projects/policyengine-simulation-gateway"] r_sim_quality_pyright_libraries_and_entry("sim.quality.pyright-libraries-and-entry (enforced)") e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry["pyright in libs/policyengine-simulation-contract, libs/policyengine-simulation-observability, libs/policyengine-stage12-persistence, libs/policyengine-fastapi, projects/policyengine-simulation-entry"] diff --git a/rules/baselines/pyright-executor.txt b/rules/baselines/pyright-executor.txt deleted file mode 100644 index aafb93a73..000000000 --- a/rules/baselines/pyright-executor.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Tolerated Pyright errors in projects/policyengine-simulation-executor. -# This list may only shrink. -# -# Rule sim.quality.pyright-executor. Each entry is -# :: :: , one line per -# error, sorted. After fixing errors, remove the entries that no longer -# occur with -# python -m scripts.rules pyright projects/policyengine-simulation-executor --write-baseline -# then set the rule's [rule.baseline] count in rules/registry.toml to match. diff --git a/rules/baselines/pyright-gateway.txt b/rules/baselines/pyright-gateway.txt deleted file mode 100644 index b407bea2d..000000000 --- a/rules/baselines/pyright-gateway.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Tolerated Pyright errors in projects/policyengine-simulation-gateway. -# This list may only shrink. -# -# Rule sim.quality.pyright-gateway. Each entry is -# :: :: , one line per -# error, sorted. After fixing errors, remove the entries that no longer -# occur with -# python -m scripts.rules pyright projects/policyengine-simulation-gateway --write-baseline -# then set the rule's [rule.baseline] count in rules/registry.toml to match. diff --git a/rules/registry.toml b/rules/registry.toml index bd7c05e0a..8974e5c51 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -1101,11 +1101,11 @@ invocation = "unittest discover -s scripts/rules/tests" [[rule]] id = "sim.quality.pyright-gateway" category = "quality" -statement = "The simulation gateway has no Pyright errors beyond those its baseline lists, and the baseline lists only errors that still occur, so it only shrinks; checked with the pinned Pyright version in the gateway's own locked environment." -rationale = "docs/engineering/skills/type-checking.md#baselines" +statement = "The simulation gateway has no Pyright errors, checked with the pinned Pyright version in the gateway's own locked environment." +rationale = "docs/engineering/skills/type-checking.md#rules" scope = ["projects/policyengine-simulation-gateway/src/**/*.py"] tier = "mechanical" -status = "ratcheted" +status = "enforced" autofix = false [rule.enforcer] kind = "pyright" @@ -1114,9 +1114,6 @@ projects = ["projects/policyengine-simulation-gateway"] workflow = ".github/workflows/pr.yml" job = "pyright" invocation = "python -m scripts.rules pyright" -[rule.baseline] -file = "rules/baselines/pyright-gateway.txt" -count = 0 [rule.proof] bad_fixture = "rules/fixtures/sim.quality.pyright-gateway/bad" good_fixture = "rules/fixtures/sim.quality.pyright-gateway/good" @@ -1128,11 +1125,11 @@ invocation = "unittest discover -s scripts/rules/tests" [[rule]] id = "sim.quality.pyright-executor" category = "quality" -statement = "The simulation executor has no Pyright errors beyond those its baseline lists, and the baseline lists only errors that still occur, so it only shrinks; checked with the pinned Pyright version in the executor's own locked environment." -rationale = "docs/engineering/skills/type-checking.md#baselines" +statement = "The simulation executor has no Pyright errors, checked with the pinned Pyright version in the executor's own locked environment." +rationale = "docs/engineering/skills/type-checking.md#rules" scope = ["projects/policyengine-simulation-executor/src/**/*.py"] tier = "mechanical" -status = "ratcheted" +status = "enforced" autofix = false [rule.enforcer] kind = "pyright" @@ -1141,9 +1138,6 @@ projects = ["projects/policyengine-simulation-executor"] workflow = ".github/workflows/pr.yml" job = "pyright" invocation = "python -m scripts.rules pyright" -[rule.baseline] -file = "rules/baselines/pyright-executor.txt" -count = 0 [rule.proof] bad_fixture = "rules/fixtures/sim.quality.pyright-executor/bad" good_fixture = "rules/fixtures/sim.quality.pyright-executor/good" From 133c4d08dfc8c336245ac8eb7747a154070f015e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:28:05 +0400 Subject: [PATCH 3/3] Say the gateway and executor proofs use excerpts of the real reports The proof comments called the bad fixtures the packages' real errors before their fixes; they are three-error excerpts of those 81- and 178-error reports. Co-Authored-By: Claude Opus 5.5 --- scripts/rules/tests/test_pyright.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/rules/tests/test_pyright.py b/scripts/rules/tests/test_pyright.py index 7e319b4c7..7fa74dc8c 100644 --- a/scripts/rules/tests/test_pyright.py +++ b/scripts/rules/tests/test_pyright.py @@ -190,8 +190,8 @@ def test_rule_proof_pyright_libraries_and_entry(self) -> None: def test_rule_proof_pyright_gateway(self) -> None: rule_id = "sim.quality.pyright-gateway" - # The gateway's real errors before its fix: variables assigned inside - # a span, which Pyright took to swallow exceptions. + # Three of the gateway's 81 real errors before its fixes: variables + # assigned inside a span, which Pyright took to swallow exceptions. bad = proof_comparison(rule_id, "bad") self.assertEqual( @@ -207,9 +207,9 @@ def test_rule_proof_pyright_gateway(self) -> None: def test_rule_proof_pyright_executor(self) -> None: rule_id = "sim.quality.pyright-executor" - # The executor's real errors before its fixes: modal attributes - # unknown while src/modal shadowed the library, and a variable - # assigned inside a span. + # Three of the executor's 178 real errors before its fixes: modal + # attributes unknown while src/modal shadowed the library, and a + # variable assigned inside a span. bad = proof_comparison(rule_id, "bad") self.assertEqual(