diff --git a/.github/scripts/cloud-run-simulation-entry-smoke.sh b/.github/scripts/cloud-run-simulation-entry-smoke.sh index 84c0726da..d34aa0de5 100755 --- a/.github/scripts/cloud-run-simulation-entry-smoke.sh +++ b/.github/scripts/cloud-run-simulation-entry-smoke.sh @@ -25,6 +25,12 @@ previous_revision="${3:-}" # and model. # This is the identity the gateway reports from the routing registry; the # numerical release sentinel checks what the worker computes. +# REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION=true (candidate checks only): +# /ready must report the entrypoint's own database reachable and at the +# migration revision the image expects, which shows that the runtime secret +# reached the revision and that the migration job ran. Stable and rollback +# checks leave it unset, because an older image may face a newer schema. +require_database_revision="${REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION:-}" expected_identity="${EXPECTED_IDENTITY:-none}" expected_policyengine_version="${EXPECTED_POLICYENGINE_VERSION:-}" expected_app_name="${EXPECTED_APP_NAME:-}" @@ -39,6 +45,14 @@ case "${expected_identity}" in exit 2 ;; esac +case "${require_database_revision}" in + "" | true) ;; + *) + printf 'REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION must be true or empty, not %s\n' \ + "${require_database_revision}" >&2 + exit 2 + ;; +esac case "${expected_latest}" in "" | true) ;; *) @@ -63,6 +77,9 @@ if [ "${expected_identity}" = "required" ] || fi routing_max_attempts="${SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS:-24}" routing_retry_delay_seconds="${SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS:-5}" +# A cold candidate may not reach its database within /ready's three-second +# wait; an unreachable database is asked about again this many times. +database_ready_max_attempts="${SIMULATION_DATABASE_READY_MAX_ATTEMPTS:-6}" if ! [[ "${routing_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then printf 'SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS must be a positive integer\n' >&2 @@ -72,6 +89,10 @@ if ! [[ "${routing_retry_delay_seconds}" =~ ^[0-9]+$ ]]; then printf 'SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS must be a non-negative integer\n' >&2 exit 2 fi +if ! [[ "${database_ready_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then + printf 'SIMULATION_DATABASE_READY_MAX_ATTEMPTS must be a positive integer\n' >&2 + exit 2 +fi health_headers="$(mktemp)" health_body="$(mktemp)" @@ -131,8 +152,29 @@ while true; do attempt=$((attempt + 1)) done -curl --fail --silent --show-error "${base_url}/ready" | - jq -e '.status == "ready"' >/dev/null +ready_body="$(curl --fail --silent --show-error "${base_url}/ready")" +jq -e '.status == "ready"' <<<"${ready_body}" >/dev/null +if [ "${require_database_revision}" = "true" ]; then + database_attempt=1 + while [ "${database_attempt}" -lt "${database_ready_max_attempts}" ] && + jq -e '.simulation_database.reachable == false' <<<"${ready_body}" >/dev/null; do + printf '%s: the simulation database was not reached; asking again (%s/%s)\n' \ + "${base_url}" "${database_attempt}" "${database_ready_max_attempts}" >&2 + sleep "${routing_retry_delay_seconds}" + ready_body="$(curl --fail --silent --show-error "${base_url}/ready")" + database_attempt=$((database_attempt + 1)) + done +fi +if [ "${require_database_revision}" = "true" ] && + ! jq -e ' + .simulation_database.reachable == true + and (.simulation_database.revision | type == "string") + and .simulation_database.revision == .simulation_database.expected_revision + ' <<<"${ready_body}" >/dev/null; then + printf '%s: the simulation database is not at the expected revision: %s\n' \ + "${base_url}" "$(jq -c '.simulation_database' <<<"${ready_body}")" >&2 + exit 1 +fi curl --fail --silent --show-error "${base_url}/versions" --output "${versions_body}" jq -e 'type == "object"' "${versions_body}" >/dev/null diff --git a/.github/scripts/deploy-cloud-run-simulation-entry.sh b/.github/scripts/deploy-cloud-run-simulation-entry.sh index 4320514ff..0956ae9a5 100755 --- a/.github/scripts/deploy-cloud-run-simulation-entry.sh +++ b/.github/scripts/deploy-cloud-run-simulation-entry.sh @@ -58,6 +58,28 @@ resolve_secret_reference() { printf '%s:%s' "${reference}" "${highest_enabled_version}" } +runtime_secrets=() + +# Resolve one secret per assignment: an array assignment takes its exit status +# from its last command substitution only, so resolving several secrets in +# one would let an earlier failure pass unnoticed. +add_runtime_secret() { + local variable_name="$1" + local resolved_reference + resolved_reference="$(resolve_secret_reference "$2")" + runtime_secrets+=("${variable_name}=${resolved_reference}") +} + +# The Stage 12 comparison path's own wrapper, removed with it, so the main +# path's wrapper can change without touching the Stage 12 block. Both use the +# secret lookups above, which predate the split. +add_stage12_runtime_secret() { + local variable_name="$1" + local resolved_reference + resolved_reference="$(resolve_secret_reference "$2")" + runtime_secrets+=("${variable_name}=${resolved_reference}") +} + required_variables=( PROJECT_ID REGION @@ -77,6 +99,7 @@ required_variables=( OLD_GATEWAY_AUTH_AUDIENCE_VALUE OLD_GATEWAY_AUTH_CLIENT_ID_VALUE OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME + SIMULATION_DATABASE_URL_SECRET_NAME OBSERVABILITY_SERVICE_NAMESPACE OBSERVABILITY_TRACE_PROJECT_ID OTEL_EXPORTER_OTLP_ENDPOINT @@ -115,11 +138,11 @@ export SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE="${SIMULATION_ENTRYPOINT_ [[ "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "0" || "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "1" ]] \ || fail "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE must be 0 or 1" -runtime_secrets=( - "OLD_GATEWAY_AUTH_CLIENT_SECRET=$( - resolve_secret_reference "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}" - )" -) +add_runtime_secret OLD_GATEWAY_AUTH_CLIENT_SECRET \ + "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}" +# The runtime role's URL for the entrypoint's own simulation schema. The +# migration job, which uses a separate migrator URL, runs before this deploy. +add_runtime_secret SIMULATION_DATABASE_URL "${SIMULATION_DATABASE_URL_SECRET_NAME}" if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then stage12_variables=( @@ -138,11 +161,9 @@ if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then || fail "STAGE12_ARTIFACT_BUCKET_VALUE is invalid" validate_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}" - runtime_secrets+=( - "MODAL_TOKEN_ID=$(resolve_secret_reference "${MODAL_TOKEN_ID_SECRET_NAME}")" - "MODAL_TOKEN_SECRET=$(resolve_secret_reference "${MODAL_TOKEN_SECRET_SECRET_NAME}")" - "STAGE12_DATABASE_URL=$(resolve_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}")" - ) + add_stage12_runtime_secret MODAL_TOKEN_ID "${MODAL_TOKEN_ID_SECRET_NAME}" + add_stage12_runtime_secret MODAL_TOKEN_SECRET "${MODAL_TOKEN_SECRET_SECRET_NAME}" + add_stage12_runtime_secret STAGE12_DATABASE_URL "${STAGE12_DATABASE_URL_SECRET_NAME}" fi runtime_environment_file="${RUNNER_TEMP}/simulation-entry-runtime-environment.yaml" diff --git a/.github/scripts/migrate-simulation-database.sh b/.github/scripts/migrate-simulation-database.sh new file mode 100755 index 000000000..88bc3d84e --- /dev/null +++ b/.github/scripts/migrate-simulation-database.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Bring the simulation schema to the chain's head, or require it to be there. +# Run from libs/policyengine-simulation-persistence, whose locked environment +# (with the migrations extra) provides Alembic. migrations/env.py reads the +# migrator URL from SIMULATION_MIGRATION_DATABASE_URL; this script never +# prints it. +# +# SIMULATION_MIGRATION_MODE: +# upgrade apply every pending revision, then require the database at the +# head (deployments from the default branch); +# verify apply nothing and only require the database at this checkout's +# head, so a deployment dispatched from another branch can never +# apply that branch's revisions. + +set -euo pipefail +set +x + +if [[ -z "${SIMULATION_MIGRATION_DATABASE_URL:-}" ]]; then + printf 'SIMULATION_MIGRATION_DATABASE_URL must name the simulation database\n' >&2 + exit 1 +fi + +mode="${SIMULATION_MIGRATION_MODE:-}" +case "${mode}" in + upgrade | verify) ;; + *) + printf 'SIMULATION_MIGRATION_MODE must be upgrade or verify, not %s\n' "${mode:-empty}" >&2 + exit 1 + ;; +esac + +alembic_config="../../migrations/alembic.ini" + +if [[ "${mode}" == "upgrade" ]]; then + uv run --frozen --extra migrations alembic -c "${alembic_config}" upgrade head +fi +uv run --frozen --extra migrations alembic -c "${alembic_config}" current --check-heads diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0c27df2f1..ac46f768b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,7 +16,7 @@ jobs: # every job passed. See docs/engineering/skills/github-prs.md. name: CI complete if: ${{ always() }} - needs: [test, lint, pyright, quality-guards, docker-build, test-integration] + needs: [test, simulation-database, lint, pyright, quality-guards, docker-build, test-integration] runs-on: ubuntu-latest permissions: contents: read @@ -51,6 +51,7 @@ jobs: - projects/policyengine-simulation-gateway - libs/policyengine-simulation-contract - libs/policyengine-simulation-observability + - libs/policyengine-simulation-persistence - libs/policyengine-stage12-persistence - libs/policyengine-fastapi @@ -75,6 +76,54 @@ jobs: working-directory: ${{ matrix.project }} run: uv run --frozen pytest tests/ -v + simulation-database: + name: Simulation database + # Runs the simulation schema's migration chain (upgrade, alembic check, + # downgrade, upgrade) and the persistence tests that need a database, on a + # fresh PostgreSQL. Full CI runs only on the lowest unmerged pull request + # in a stack. + if: ${{ !github.event.pull_request.stack || github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref }} + runs-on: ubuntu-latest + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: simulation_ci + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U postgres" + --health-interval=5s + --health-timeout=5s + --health-retries=20 + env: + SIMULATION_TEST_DATABASE_URL: postgresql://postgres:simulation_ci@127.0.0.1:5432/postgres + # A missing server fails the tests instead of skipping them. + SIMULATION_REQUIRE_DATABASE_TESTS: "1" + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Install dependencies + working-directory: libs/policyengine-simulation-persistence + run: uv sync --extra test --frozen + + - name: Run the migration chain and persistence tests on PostgreSQL + working-directory: libs/policyengine-simulation-persistence + run: uv run --frozen pytest tests/ -v -m postgres + lint: name: Lint # Full CI runs only on the lowest unmerged pull request in a stack. @@ -157,6 +206,7 @@ jobs: package: - libs/policyengine-simulation-contract - libs/policyengine-simulation-observability + - libs/policyengine-simulation-persistence - libs/policyengine-stage12-persistence - libs/policyengine-fastapi - projects/policyengine-simulation-entry diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 9de0a3eff..15990f525 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -153,10 +153,40 @@ jobs: GATEWAY_AUTH_REQUIRED: ${{ vars.GATEWAY_AUTH_REQUIRED }} run: ../../.github/scripts/modal-sync-secrets.sh "${{ inputs.modal_environment }}" + migrate_simulation_database: + # Applies the simulation schema's Alembic chain before the entrypoint + # candidate deploys. The previous revision keeps serving meanwhile, so + # every migration must work with the release before it. + name: Migrate the simulation database schema + needs: prepare + runs-on: ubuntu-latest + environment: ${{ inputs.release_environment }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - uses: astral-sh/setup-uv@v8.1.0 + + - name: Install migration dependencies + working-directory: libs/policyengine-simulation-persistence + run: uv sync --frozen --extra migrations + + - name: Apply simulation schema migrations + working-directory: libs/policyengine-simulation-persistence + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + # Only a deployment from main applies revisions. One dispatched from + # another branch requires the database to be at its chain's head. + SIMULATION_MIGRATION_MODE: ${{ github.ref == 'refs/heads/main' && 'upgrade' || 'verify' }} + run: ../../.github/scripts/migrate-simulation-database.sh + deploy_entrypoint: name: Deploy and unit test Cloud Run entrypoint - if: ${{ always() && needs.prepare.result == 'success' && ((inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'success') || (!inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'skipped')) }} - needs: [prepare, configure_stage12_v2] + if: ${{ always() && needs.prepare.result == 'success' && needs.migrate_simulation_database.result == 'success' && ((inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'success') || (!inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'skipped')) }} + needs: [prepare, configure_stage12_v2, migrate_simulation_database] runs-on: ubuntu-latest environment: ${{ inputs.release_environment }} permissions: @@ -237,6 +267,7 @@ jobs: OLD_GATEWAY_AUTH_AUDIENCE_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_AUDIENCE }} OLD_GATEWAY_AUTH_CLIENT_ID_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_CLIENT_ID }} OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME: ${{ vars.OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME }} + SIMULATION_DATABASE_URL_SECRET_NAME: ${{ vars.SIMULATION_DATABASE_URL_SECRET_NAME }} STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }} STAGE12_ARTIFACT_BUCKET_VALUE: ${{ vars.STAGE12_ARTIFACT_BUCKET }} STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }} @@ -744,6 +775,8 @@ jobs: EXPECTED_UK_DATASET_SHA256: ${{ inputs.deploy_existing_stack && needs.prepare.outputs.uk_dataset_sha256 || '' }} # Latest must route to this release only when it should advance it. EXPECTED_LATEST: ${{ inputs.deploy_existing_stack && needs.update_routing.outputs.advances_latest == 'true' && 'true' || '' }} + # The candidate reaches its own database at the migrated revision. + REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION: "true" run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.deploy_entrypoint.outputs.revision }}" - name: Download the numerical release sentinel diff --git a/AGENTS.md b/AGENTS.md index 12ba40fb5..7c410cfdf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,7 +120,11 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. ### Stage 12 persistence -- Checked: `sim.stage12.deployment-scope`, `sim.stage12.deployment-uses-api-schema`, `sim.stage12.no-migration-chain`, `sim.stage12.no-production-user-associations`, `sim.stage12.no-schema-ddl`, `sim.stage12.persistence-typed-dml-only`, `sim.stage12.production-lock-has-db-runtime` +- Checked: `sim.stage12.deployment-scope`, `sim.stage12.deployment-uses-api-schema`, `sim.stage12.no-production-user-associations`, `sim.stage12.no-schema-ddl`, `sim.stage12.persistence-typed-dml-only`, `sim.stage12.production-lock-has-db-runtime` + +### Simulation persistence + +- Checked: `sim.persistence.executor-independent`, `sim.persistence.migrate-before-entrypoint`, `sim.persistence.migration-cycle`, `sim.persistence.no-modal-database-credentials`, `sim.persistence.one-migration-chain` ### Deployment pipeline diff --git a/Makefile b/Makefile index 94484d22c..7a0aea572 100644 --- a/Makefile +++ b/Makefile @@ -89,6 +89,7 @@ test: projects/policyengine-simulation-gateway \ libs/policyengine-simulation-contract \ libs/policyengine-simulation-observability \ + libs/policyengine-simulation-persistence \ libs/policyengine-stage12-persistence \ libs/policyengine-fastapi; do \ echo "Testing $$proj..."; \ diff --git a/docs/engineering/skills/database-access.md b/docs/engineering/skills/database-access.md index 1ed83a886..8448cb941 100644 --- a/docs/engineering/skills/database-access.md +++ b/docs/engineering/skills/database-access.md @@ -21,31 +21,32 @@ It builds on the Stage 12 rules: - `sim.stage12.no-schema-ddl`: the simulation runtimes and the Stage 12 persistence library contain no schema DDL. Its `DDL_PATTERN` regex is the one this rule matches against every source directory's string literals. -- `sim.stage12.no-migration-chain`: this repository has no migration chain - of its own; policyengine-api migrates the Stage 12 tables. +- `sim.persistence.one-migration-chain`: the repository's one Alembic chain, + at `migrations/`, owns the Simulation Entrypoint's `simulation` schema; + the Public API migrates the Stage 12 tables. - `sim.stage12.persistence-typed-dml-only` and `sim.stage12.deployment-uses-api-schema`: the Stage 12 persistence library uses typed SQLAlchemy Core DML only, with no raw SQL text. ## Who owns the schema -- **Today**, policyengine-api's Alembic migrations own the only tables this - repository uses, the Stage 12 comparison tables - (`sim.stage12.no-migration-chain`). This repository's code reads and - writes their rows, and deployment validation compares their live schema - with its SQLAlchemy mappings; nothing here changes them. -- **Once #847 merges**, this repository's own Alembic chain at `migrations/` - owns the `simulation` schema, and the Stage 12 tables stay with - policyengine-api. +- **The `simulation` schema** belongs to this repository's own Alembic chain + at `migrations/` (`sim.persistence.one-migration-chain`). The tables are + declared in `libs/policyengine-simulation-persistence`, whose README + describes the chain, the database roles and how to add a revision. +- **The Stage 12 comparison tables** belong to the Public API's Alembic + migrations. This repository's code reads and writes their rows, and + deployment validation compares their live schema with its SQLAlchemy + mappings; nothing here changes them. -The rule holds either way: whichever chain owns a table, only that chain's -migrations change it. +Either way, only the chain that owns a table changes it. ## What the guard checks `projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py` parses every Python file under `projects/*/src` (including the executor's -`src/modal`), `libs/*/src` and `.github/scripts`, and runs in the entry +`src/modal`), `libs/*/src`, `.github/scripts` and the repository's +`migrations/` chain, and runs in the entry project's `Test` job. The checks live in `database_access_support.py` next to it. It fails on: diff --git a/docs/engineering/skills/type-checking.md b/docs/engineering/skills/type-checking.md index 56a444b58..b1b8abbd2 100644 --- a/docs/engineering/skills/type-checking.md +++ b/docs/engineering/skills/type-checking.md @@ -11,9 +11,9 @@ Every uv package is type-checked with Pyright on every pull request, by the - `sim.quality.pyright-libraries-and-entry`, `sim.quality.pyright-gateway`, `sim.quality.pyright-executor` and `sim.quality.pyright-apis-integ`: the - contract, observability, Stage 12 persistence and fastapi libraries, the - simulation entry, the gateway, the executor and the generated-client - integration tests have no Pyright errors. + contract, observability, simulation persistence, Stage 12 persistence and + fastapi libraries, the simulation entry, the gateway, the executor and the + generated-client integration tests have no Pyright errors. - `sim.quality.every-package-type-checked`: a Pyright rule lists every uv package under `libs/` and `projects/`. When you add a package, give it a `[tool.pyright] include`, add it to a Pyright rule's `enforcer.projects`, and diff --git a/docs/generated/rules.md b/docs/generated/rules.md index b2f783267..5d470dda9 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 53 | 2 | 0 | 14 | 69 | +| mechanical | 57 | 2 | 0 | 14 | 73 | | metric | 1 | 0 | 0 | 0 | 1 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 56 of 70 rules. +**Blocking:** 60 of 74 rules. ## Gaps @@ -42,7 +42,6 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf - `sim.contracts.clients-publish-deployed-commit` - `sim.contracts.full-integration-beta-only` - `sim.contracts.gateway-openapi-golden` -- `sim.deploy.container-context-minimal` - `sim.deploy.gcloud-without-terraform` - `sim.deploy.numerical-sentinel-gate` - `sim.deploy.permission-enforcement-wired` @@ -53,11 +52,9 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf - `sim.deploy.shell-scripts-valid` - `sim.deploy.smoke-tests-strict` - `sim.deploy.traffic-changes-exact` -- `sim.deploy.workflow-shell-safety` - `sim.process.rules-registry` - `sim.stage12.deployment-scope` - `sim.stage12.deployment-uses-api-schema` -- `sim.stage12.no-migration-chain` - `sim.stage12.no-production-user-associations` - `sim.stage12.persistence-typed-dml-only` - `sim.stage12.production-lock-has-db-runtime` @@ -103,7 +100,6 @@ flowchart LR | --- | --- | --- | --- | --- | --- | --- | --- | | `sim.stage12.deployment-scope` | Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources or runs the existing stack's complete integration suite; the Stage 12 worker can request a Google OIDC token. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py` | `pr.yml:test` | **unproven** | | `sim.stage12.deployment-uses-api-schema` | Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.stage12.no-migration-chain` | This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-production-user-associations` | Stage 12 runtime modules never create production user associations (user_simulations or user_reports). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-schema-ddl` | Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | yes | | `sim.stage12.persistence-typed-dml-only` | The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. | mechanical | enforced | yes | pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py` | `pr.yml:test` | **unproven** | @@ -116,9 +112,8 @@ flowchart LR j_CI_pr_yml_test["CI pr.yml:test"] r_sim_stage12_deployment_uses_api_schema("sim.stage12.deployment-uses-api-schema (enforced)") e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] - r_sim_stage12_no_migration_chain("sim.stage12.no-migration-chain (enforced)") - e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py["pytest projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py"] r_sim_stage12_no_production_user_associations("sim.stage12.no-production-user-associations (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py["pytest projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py"] r_sim_stage12_no_schema_ddl("sim.stage12.no-schema-ddl (enforced)") r_sim_stage12_persistence_typed_dml_only("sim.stage12.persistence-typed-dml-only (enforced)") e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py["pytest libs/policyengine-stage12-persistence/tests/test_source_boundaries.py"] @@ -128,9 +123,8 @@ flowchart LR e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py --> j_CI_pr_yml_test r_sim_stage12_deployment_uses_api_schema --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test - r_sim_stage12_no_migration_chain --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py - e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py --> j_CI_pr_yml_test r_sim_stage12_no_production_user_associations --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py --> j_CI_pr_yml_test r_sim_stage12_no_schema_ddl --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py r_sim_stage12_persistence_typed_dml_only --> e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py --> j_CI_pr_yml_test @@ -138,11 +132,43 @@ flowchart LR e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py --> j_CI_pr_yml_test ``` +## Simulation persistence + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.persistence.executor-independent` | The Modal executor and gateway never declare, lock or import policyengine-simulation-persistence; only the Simulation Entrypoint and the migration job install it. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py` | `pr.yml:test` | yes | +| `sim.persistence.migrate-before-entrypoint` | Each deployment migrates the simulation schema in its own job, the only one that receives SIMULATION_MIGRATION_DATABASE_URL, before the entrypoint candidate deploys, applying revisions only from main and otherwise requiring the database at the chain's head; the entrypoint receives SIMULATION_DATABASE_URL as a pinned Secret Manager version; and the candidate smoke requires the database at the image's revision. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py` | `pr.yml:test` | yes | +| `sim.persistence.migration-cycle` | On a fresh PostgreSQL, the simulation chain upgrades to its head, passes alembic check, downgrades to base and upgrades again, run by a non-superuser migrator, and the runtime role can read the revision but cannot change the schema. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_migration_cycle.py` | `pr.yml:simulation-database` | yes | +| `sim.persistence.no-modal-database-credentials` | No Modal secret, Modal deployment or Modal application receives a database URL other than the comparison path's own STAGE12_DATABASE_URL, which Modal may hold until the comparison path is removed; in particular never SIMULATION_DATABASE_URL or SIMULATION_MIGRATION_DATABASE_URL. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py` | `pr.yml:test` | yes | +| `sim.persistence.one-migration-chain` | The simulation schema changes only through the one Alembic chain at migrations/, configured by migrations/alembic.ini and versioned in simulation.alembic_version; no other Alembic configuration, environment or revision directory exists. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py` | `pr.yml:test` | yes | + +```mermaid +flowchart LR + r_sim_persistence_executor_independent("sim.persistence.executor-independent (enforced)") + e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py["pytest libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_persistence_migrate_before_entrypoint("sim.persistence.migrate-before-entrypoint (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_simulation_database_deployment_py["pytest projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py"] + r_sim_persistence_migration_cycle("sim.persistence.migration-cycle (enforced)") + e_pytest_libs_policyengine_simulation_persistence_tests_test_migration_cycle_py["pytest libs/policyengine-simulation-persistence/tests/test_migration_cycle.py"] + j_CI_pr_yml_simulation_database["CI pr.yml:simulation-database"] + r_sim_persistence_no_modal_database_credentials("sim.persistence.no-modal-database-credentials (enforced)") + r_sim_persistence_one_migration_chain("sim.persistence.one-migration-chain (enforced)") + r_sim_persistence_executor_independent --> e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py + e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py --> j_CI_pr_yml_test + r_sim_persistence_migrate_before_entrypoint --> e_pytest_projects_policyengine_simulation_entry_tests_test_simulation_database_deployment_py + e_pytest_projects_policyengine_simulation_entry_tests_test_simulation_database_deployment_py --> j_CI_pr_yml_test + r_sim_persistence_migration_cycle --> e_pytest_libs_policyengine_simulation_persistence_tests_test_migration_cycle_py + e_pytest_libs_policyengine_simulation_persistence_tests_test_migration_cycle_py --> j_CI_pr_yml_simulation_database + r_sim_persistence_no_modal_database_credentials --> e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py + r_sim_persistence_one_migration_chain --> e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py +``` + ## Deployment pipeline | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | -| `sim.deploy.container-context-minimal` | Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.container-context-minimal` | Container builds exclude local environments and unrelated projects but include every local library the entrypoint installs, and the entrypoint container builds for the requested target platform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | yes | | `sim.deploy.docker-images-build` | The simulation entry and executor Docker images build. | mechanical | enforced | yes | CI job `docker-build` | `pr.yml:docker-build` | — | | `sim.deploy.gcloud-without-terraform` | Deployment uses the gcloud-based workflow, not Terraform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.image-smoke-covers-every-image-lib` | The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. | mechanical | doc-only | no | nothing | — | — | @@ -156,7 +182,7 @@ flowchart LR | `sim.deploy.shell-scripts-valid` | Deployment shell scripts have valid shell syntax. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.smoke-tests-strict` | Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.traffic-changes-exact` | Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.workflow-shell-safety` | Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.workflow-shell-safety` | Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, pins secret versions, stops before deploying when any runtime secret cannot be resolved, and resolves the Stage 12 secrets with the Stage 12 block's own helper. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py` | `pr.yml:test` | yes | ```mermaid flowchart LR @@ -310,7 +336,7 @@ flowchart LR | `sim.quality.pyright-apis-integ` | The generated-client integration-test project, apis-integ, has no Pyright errors, checked with the pinned Pyright version in the project's own locked environment. | mechanical | enforced | yes | pyright in `projects/policyengine-apis-integ` | `pr.yml:pyright` | yes | | `sim.quality.pyright-executor` | The simulation executor has no Pyright errors, checked with the pinned Pyright version in the executor's own locked environment. | mechanical | enforced | yes | pyright in `projects/policyengine-simulation-executor` | `pr.yml:pyright` | yes | | `sim.quality.pyright-gateway` | The simulation gateway has no Pyright errors, checked with the pinned Pyright version in the gateway's own locked environment. | mechanical | enforced | yes | pyright in `projects/policyengine-simulation-gateway` | `pr.yml:pyright` | yes | -| `sim.quality.pyright-libraries-and-entry` | The contract, observability, Stage 12 persistence and fastapi libraries and the simulation entry have no Pyright errors, checked with the pinned Pyright version in each package's own locked environment. | mechanical | enforced | yes | pyright in `libs/policyengine-simulation-contract`, `libs/policyengine-simulation-observability`, `libs/policyengine-stage12-persistence`, `libs/policyengine-fastapi`, `projects/policyengine-simulation-entry` | `pr.yml:pyright` | yes | +| `sim.quality.pyright-libraries-and-entry` | The contract, observability, simulation persistence, Stage 12 persistence and fastapi libraries and the simulation entry have no Pyright errors, checked with the pinned Pyright version in each package's own locked environment. | mechanical | enforced | yes | pyright in `libs/policyengine-simulation-contract`, `libs/policyengine-simulation-observability`, `libs/policyengine-simulation-persistence`, `libs/policyengine-stage12-persistence`, `libs/policyengine-fastapi`, `projects/policyengine-simulation-entry` | `pr.yml:pyright` | yes | | `sim.quality.python-file-length` | No tracked Python file has more than 800 lines, or more than 1,500 lines if it is test code: a file under a tests or fixtures directory, or a conftest.py. | metric | enforced | yes | quality guard `file-length` | `pr.yml:quality-guards` | yes | | `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — | @@ -343,7 +369,7 @@ flowchart LR r_sim_quality_pyright_gateway("sim.quality.pyright-gateway (enforced)") e_pyright_in_projects_policyengine_simulation_gateway["pyright in projects/policyengine-simulation-gateway"] r_sim_quality_pyright_libraries_and_entry("sim.quality.pyright-libraries-and-entry (enforced)") - e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry["pyright in libs/policyengine-simulation-contract, libs/policyengine-simulation-observability, libs/policyengine-stage12-persistence, libs/policyengine-fastapi, projects/policyengine-simulation-entry"] + e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_simulation_persistence__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry["pyright in libs/policyengine-simulation-contract, libs/policyengine-simulation-observability, libs/policyengine-simulation-persistence, libs/policyengine-stage12-persistence, libs/policyengine-fastapi, projects/policyengine-simulation-entry"] r_sim_quality_python_file_length("sim.quality.python-file-length (enforced)") e_quality_guard_file_length["quality guard file-length"] r_sim_quality_ruff_format_src("sim.quality.ruff-format-src (enforced)") @@ -377,8 +403,8 @@ flowchart LR e_pyright_in_projects_policyengine_simulation_executor --> j_CI_pr_yml_pyright r_sim_quality_pyright_gateway --> e_pyright_in_projects_policyengine_simulation_gateway e_pyright_in_projects_policyengine_simulation_gateway --> j_CI_pr_yml_pyright - r_sim_quality_pyright_libraries_and_entry --> e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry - e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry --> j_CI_pr_yml_pyright + r_sim_quality_pyright_libraries_and_entry --> e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_simulation_persistence__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry + e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_simulation_persistence__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry --> j_CI_pr_yml_pyright r_sim_quality_python_file_length --> e_quality_guard_file_length e_quality_guard_file_length --> j_CI_pr_yml_quality_guards r_sim_quality_ruff_format_src --> e_ruff_format diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md new file mode 100644 index 000000000..ab82daf90 --- /dev/null +++ b/libs/policyengine-simulation-persistence/README.md @@ -0,0 +1,114 @@ +# PolicyEngine simulation persistence + +SQLAlchemy Core tables and database connections for the Simulation +Entrypoint's own Postgres schema, `simulation`. The Simulation Entrypoint owns +every table in that schema; no other service creates, changes or writes them. + +The temporary Stage 12 comparison tables are not part of this schema: they are +owned and migrated by the Public API, and `libs/policyengine-stage12-persistence` +writes them with DML only. + +## Ownership rules + +These rules are registered in `rules/registry.toml` and checked by +`tests/test_ownership_boundaries.py`: + +- **One migration chain** (`sim.persistence.one-migration-chain`). Schema + changes are made only by the Alembic chain in the repository's + `migrations/` directory, configured by `migrations/alembic.ini`. It keeps its + version table at `simulation.alembic_version`. No other Alembic + configuration, environment or revision directory exists in the repository. + Runtime code never changes a schema + (`sim.boundaries.sqlalchemy-and-alembic-only`). +- **The executor does not depend on persistence** + (`sim.persistence.executor-independent`). The Modal executor and gateway + never declare, lock or import this library. Only the Simulation Entrypoint + and the migration job install it. +- **No database credentials in Modal** + (`sim.persistence.no-modal-database-credentials`). No Modal secret, Modal + deployment or Modal application source receives or names a database URL: + neither a variable named like `*DATABASE_URL*` nor a written-out + `postgresql://` URL, whether in a step's, job's or workflow's environment + or in a script such a step runs. The one exception is the Stage 12 + comparison's own `STAGE12_DATABASE_URL` (and the name of the secret that + holds it), until the comparison path is removed. In particular, + `SIMULATION_DATABASE_URL` and `SIMULATION_MIGRATION_DATABASE_URL` never + reach Modal. + +## Database roles + +| Role | Used by | Privileges | +| --- | --- | --- | +| `simulation_migrator` | The deploy workflow's migration job, through `SIMULATION_MIGRATION_DATABASE_URL` | Creates the `simulation` schema if it is missing, and owns every object in it | +| `simulation_runtime` | The Simulation Entrypoint, through `SIMULATION_DATABASE_URL` | `USAGE` on the schema, `SELECT` on the version table, and `SELECT`, `INSERT` and `UPDATE` on the tables the migrations create | + +The initial revision grants the runtime role its privileges, including default +privileges on tables the migrator creates later. Both roles must exist before +the first migration runs; an operator creates them. The migrator needs either +`CREATE` on the database or a `simulation` schema created in advance and owned +by it: the environment creates the schema only when it is missing, because +PostgreSQL checks `CREATE` on the database even for +`CREATE SCHEMA IF NOT EXISTS`. + +## Deployment + +On every deployment, the migration job (`migrate_simulation_database` in +`.github/workflows/simulation-deploy.reusable.yml`) runs +`.github/scripts/migrate-simulation-database.sh` before the Simulation +Entrypoint's candidate revision is deployed +(`sim.persistence.migrate-before-entrypoint`). From `main` it runs +`alembic upgrade head` and then `alembic current --check-heads`; from any other +branch it runs only the head check, so a deployment dispatched from an unmerged +branch fails instead of applying that branch's revisions. It is the only job +that receives `SIMULATION_MIGRATION_DATABASE_URL`. If it fails, the entrypoint +is not deployed. The Simulation Entrypoint's readiness check (`GET /ready`) reports +the database's revision next to `HEAD_REVISION`, the revision its image +expects, and the candidate check requires the two to match. + +## Connections + +`create_simulation_engine` builds the runtime engine from the canonical +`postgresql://` URL the deployment stores, selecting the psycopg 3 driver. It +keeps a small pool, checks connections before use, and never prepares +server-side statements, so the same URL works through Supabase's session and +transaction poolers. A malformed URL raises `ValueError` without repeating the +URL, which holds a password. + +`read_schema_revision` returns the migration revision the database has applied, +or `None` before the first migration. `HEAD_REVISION` is the newest revision +this code knows. + +## Adding a revision + +1. Change the tables in `src/policyengine_simulation_persistence/`. +2. Add a revision under `migrations/versions/` whose `down_revision` is the + current head, and set `HEAD_REVISION` in `schema.py` to its revision ID. +3. Run the tests against a disposable PostgreSQL server (below). They run the + chain's upgrade, `alembic check`, downgrade and upgrade cycle. + +The schema creation and every pending revision apply in one transaction, so a +failed migration leaves nothing applied. A statement that waits more than ten +seconds for a lock fails the migration instead of waiting longer, because +adding a foreign key locks the Public API table it refers to and the Public +API's writes would queue behind it; the next deployment tries again. The previous release keeps serving +traffic while a deployment migrates, and a rollback moves traffic back to an +older image without downgrading the schema, so every revision must work with +the code of the release before it: add columns and tables, and remove them +only in a later release once nothing reads them. + +Only one migration PR in this chain may merge at a time; another one open at +the same time rebases onto the new head after the first merges. + +## Tests + +```bash +uv sync --frozen --extra test +uv run --frozen pytest tests/ +``` + +Tests marked `postgres` need a disposable PostgreSQL server. Point +`SIMULATION_TEST_DATABASE_URL` at its superuser, for example +`postgresql://postgres:password@127.0.0.1:5432/postgres`; each test creates and +drops its own database. Without the variable those tests skip. CI's +`Simulation database` job runs them against `postgres:17` with +`SIMULATION_REQUIRE_DATABASE_TESTS=1`, which turns a skip into a failure. diff --git a/libs/policyengine-simulation-persistence/pyproject.toml b/libs/policyengine-simulation-persistence/pyproject.toml new file mode 100644 index 000000000..28ab475b5 --- /dev/null +++ b/libs/policyengine-simulation-persistence/pyproject.toml @@ -0,0 +1,49 @@ +[project] +name = "policyengine-simulation-persistence" +version = "0.1.0" +readme = "README.md" +authors = [ + {name = "PolicyEngine", email = "hello@policyengine.org"}, +] +license = {file = "../../LICENSE"} +requires-python = ">=3.13,<3.14" +dependencies = [ + "psycopg[binary]>=3.2,<4", + "sqlalchemy>=2,<3", +] + +[project.optional-dependencies] +# The migration job installs Alembic; the Simulation Entrypoint runtime does not. +# 1.17.1 added `alembic current --check-heads`, which the migration job runs. +migrations = [ + "alembic>=1.17.1,<2", +] +test = [ + "alembic>=1.17.1,<2", + "pytest>=8.3.4", + "pytest-cov>=6.1.1", + "pyyaml>=6.0.2", +] +build = [ + "pyright>=1.1.401", +] + +[build-system] +requires = ["hatchling>=1.26"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +sources = ["src"] +only-include = ["src/policyengine_simulation_persistence"] +packages = ["src/policyengine_simulation_persistence"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] +markers = [ + "postgres: needs a disposable PostgreSQL server named by SIMULATION_TEST_DATABASE_URL", +] + +[tool.pyright] +# The pr.yml pyright job checks these paths: python -m scripts.rules pyright. +include = ["src"] diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py new file mode 100644 index 000000000..fb098c8ed --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -0,0 +1,25 @@ +"""Tables and connections for the Simulation Entrypoint's ``simulation`` schema.""" + +from policyengine_simulation_persistence.engine import ( + create_simulation_engine, + normalized_database_url, + read_schema_revision, +) +from policyengine_simulation_persistence.schema import ( + ALEMBIC_VERSION_TABLE, + HEAD_REVISION, + RUNTIME_ROLE, + SIMULATION_SCHEMA, + simulation_metadata, +) + +__all__ = [ + "ALEMBIC_VERSION_TABLE", + "HEAD_REVISION", + "RUNTIME_ROLE", + "SIMULATION_SCHEMA", + "create_simulation_engine", + "normalized_database_url", + "read_schema_revision", + "simulation_metadata", +] diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py new file mode 100644 index 000000000..58e70b3dc --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py @@ -0,0 +1,76 @@ +"""Database connections for the Simulation Entrypoint's own schema.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import create_engine, inspect, select +from sqlalchemy.engine import URL, make_url +from sqlalchemy.exc import ArgumentError + +from policyengine_simulation_persistence.schema import ( + ALEMBIC_VERSION_TABLE, + SIMULATION_SCHEMA, + alembic_version, +) + +if TYPE_CHECKING: + from sqlalchemy import Connection, Engine + +CONNECT_TIMEOUT_SECONDS = 5 +# Each Cloud Run instance serves up to 80 concurrent requests. Each database +# operation holds a connection for one short transaction, so a small pool is +# enough. +POOL_SIZE = 2 +MAX_OVERFLOW = 3 + + +def normalized_database_url(database_url: str) -> URL: + """Return ``database_url`` with the psycopg 3 driver selected. + + The stored secrets use the canonical ``postgresql://`` scheme. Raise + ValueError, without echoing the value, when the URL is missing, malformed + or incomplete. + """ + if not database_url or database_url != database_url.strip(): + msg = "Simulation database URL is missing or malformed" + raise ValueError(msg) + try: + parsed = make_url(database_url) + except (ArgumentError, ValueError): + # Raised without the parser's message, which may quote the URL. + msg = "Simulation database URL is malformed" + raise ValueError(msg) from None + if parsed.drivername != "postgresql": + msg = "Simulation database URL must use postgresql://" + raise ValueError(msg) + if not parsed.username or not parsed.host or not parsed.database: + msg = "Simulation database URL is incomplete" + raise ValueError(msg) + return parsed.set(drivername="postgresql+psycopg") + + +def create_simulation_engine(database_url: str) -> Engine: + """Build the engine the Simulation Entrypoint uses; it creates no objects.""" + return create_engine( + normalized_database_url(database_url), + pool_size=POOL_SIZE, + max_overflow=MAX_OVERFLOW, + pool_pre_ping=True, + connect_args={ + "connect_timeout": CONNECT_TIMEOUT_SECONDS, + # Never use server-side prepared statements, so the same engine + # works through Supabase's session and transaction poolers. + "prepare_threshold": None, + }, + ) + + +def read_schema_revision(connection: Connection) -> str | None: + """Return the applied Alembic revision, or None before the first migration.""" + if not inspect(connection).has_table( + ALEMBIC_VERSION_TABLE, + schema=SIMULATION_SCHEMA, + ): + return None + return connection.execute(select(alembic_version.c.version_num)).scalar() diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/schema.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/schema.py new file mode 100644 index 000000000..a7cd53c18 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/schema.py @@ -0,0 +1,44 @@ +"""Names and metadata of the Simulation Entrypoint's own Postgres schema. + +The Simulation Entrypoint owns every table in the ``simulation`` schema. One +Alembic chain, at the repository's ``migrations/`` directory, creates and +changes them, and records its revision in ``simulation.alembic_version``. +""" + +from __future__ import annotations + +from sqlalchemy import Column, MetaData, String, Table + +SIMULATION_SCHEMA = "simulation" +ALEMBIC_VERSION_TABLE = "alembic_version" +# The login role the Simulation Entrypoint runtime connects as. Migrations grant +# it schema usage and table DML; it never owns or changes schema objects. +RUNTIME_ROLE = "simulation_runtime" +# The newest revision in migrations/versions. Each migration PR updates it, and +# tests/test_schema.py fails when it differs from the chain's head. The +# readiness check reports it next to the database's revision. +HEAD_REVISION = "0001" + +NAMING_CONVENTION = { + "ix": "ix_%(table_name)s_%(column_0_N_name)s", + "uq": "uq_%(table_name)s_%(column_0_N_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + +# Every table the Simulation Entrypoint owns is registered here. +simulation_metadata = MetaData( + schema=SIMULATION_SCHEMA, + naming_convention=NAMING_CONVENTION, +) + +# Alembic creates and owns this table; it is described here, outside +# simulation_metadata, only so the runtime can read the applied revision +# without depending on Alembic. +_alembic_metadata = MetaData(schema=SIMULATION_SCHEMA) +alembic_version = Table( + ALEMBIC_VERSION_TABLE, + _alembic_metadata, + Column("version_num", String(32), primary_key=True), +) diff --git a/libs/policyengine-simulation-persistence/tests/conftest.py b/libs/policyengine-simulation-persistence/tests/conftest.py new file mode 100644 index 000000000..1726049dc --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/conftest.py @@ -0,0 +1,116 @@ +"""Disposable PostgreSQL databases for the migration and store tests. + +Tests marked ``postgres`` need a PostgreSQL server whose superuser URL is in +``SIMULATION_TEST_DATABASE_URL``. Each test gets a new database owned by a +non-superuser migrator role, so migrations run with the privileges they have in +production. Without the variable those tests skip, unless +``SIMULATION_REQUIRE_DATABASE_TESTS=1`` (set in CI), where they fail instead. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from migration_support import ( + MIGRATION_DATABASE_URL_VARIABLE, + MIGRATOR_ROLE, + TEST_ROLE_PASSWORD, +) +from policyengine_simulation_persistence.schema import RUNTIME_ROLE +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL, make_url + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Connection + +DATABASE_URL_VARIABLE = "SIMULATION_TEST_DATABASE_URL" +REQUIRE_DATABASE_VARIABLE = "SIMULATION_REQUIRE_DATABASE_TESTS" + + +@dataclass(frozen=True) +class DisposableDatabase: + """One empty database and the URLs of the roles that use it.""" + + name: str + migrator_url: str + runtime_url: str + superuser_url: str + + +def _canonical(url: URL) -> str: + return url.set(drivername="postgresql").render_as_string(hide_password=False) + + +def _ensure_login_role(connection: Connection, role_name: str) -> None: + connection.execute( + text( + "DO $$ BEGIN " + f"IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '{role_name}') " + f"THEN CREATE ROLE {role_name} LOGIN PASSWORD '{TEST_ROLE_PASSWORD}'; " + "END IF; END $$" + ) + ) + + +@pytest.fixture(scope="session") +def postgres_server_url() -> URL: + """Return the superuser URL of the test server, or skip without one.""" + configured = os.environ.get(DATABASE_URL_VARIABLE, "") + if not configured: + message = f"{DATABASE_URL_VARIABLE} names no PostgreSQL server" + if os.environ.get(REQUIRE_DATABASE_VARIABLE) == "1": + pytest.fail(message) + pytest.skip(message) + return make_url(configured).set(drivername="postgresql+psycopg") + + +@pytest.fixture +def disposable_database( + postgres_server_url: URL, + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[DisposableDatabase]: + """Create an empty database owned by the migrator, and drop it afterwards. + + ``SIMULATION_MIGRATION_DATABASE_URL`` points at it, as in the deploy job. + """ + database_name = f"simulation_test_{uuid4().hex[:12]}" + server_engine = create_engine(postgres_server_url, isolation_level="AUTOCOMMIT") + try: + with server_engine.connect() as connection: + _ensure_login_role(connection, MIGRATOR_ROLE) + _ensure_login_role(connection, RUNTIME_ROLE) + connection.execute( + text(f'CREATE DATABASE "{database_name}" OWNER {MIGRATOR_ROLE}') + ) + database = DisposableDatabase( + name=database_name, + migrator_url=_canonical( + postgres_server_url.set( + username=MIGRATOR_ROLE, + password=TEST_ROLE_PASSWORD, + database=database_name, + ) + ), + runtime_url=_canonical( + postgres_server_url.set( + username=RUNTIME_ROLE, + password=TEST_ROLE_PASSWORD, + database=database_name, + ) + ), + superuser_url=_canonical(postgres_server_url.set(database=database_name)), + ) + monkeypatch.setenv(MIGRATION_DATABASE_URL_VARIABLE, database.migrator_url) + yield database + finally: + with server_engine.connect() as connection: + connection.execute( + text(f'DROP DATABASE IF EXISTS "{database_name}" WITH (FORCE)') + ) + server_engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/migration_support.py b/libs/policyengine-simulation-persistence/tests/migration_support.py new file mode 100644 index 000000000..8efb3d13e --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/migration_support.py @@ -0,0 +1,75 @@ +"""Paths and names shared by the migration and store tests.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from alembic import command +from alembic.config import Config +from policyengine_simulation_persistence import ( + create_simulation_engine, + read_schema_revision, +) + +if TYPE_CHECKING: + from collections.abc import Callable + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +ALEMBIC_CONFIG_PATH = REPOSITORY_ROOT / "migrations" / "alembic.ini" +MIGRATION_DATABASE_URL_VARIABLE = "SIMULATION_MIGRATION_DATABASE_URL" +MIGRATOR_ROLE = "simulation_migrator" +TEST_ROLE_PASSWORD = "simulation-test-password" + + +def alembic_config() -> Config: + """Return the repository's Alembic configuration.""" + return Config(str(ALEMBIC_CONFIG_PATH)) + + +def chain_config(script_location: Path) -> Config: + """Return a configuration for the chain at ``script_location``.""" + config = Config() + config.set_main_option("script_location", str(script_location)) + return config + + +def _step_failure(name: str, step: Callable[[], object]) -> str | None: + try: + step() + except Exception as error: # noqa: BLE001 # reported as the cycle's failure + detail = str(error).strip().splitlines() + return f"{name} failed: {type(error).__name__}" + ( + f": {detail[0]}" if detail else "" + ) + return None + + +def migration_cycle_failures(config: Config, *, check: bool) -> list[str]: + """Upgrade, downgrade and upgrade the chain again; return what failed. + + With ``check``, ``alembic check`` must also find nothing to change after + each upgrade. The cycle stops at the first failure. + """ + steps: list[tuple[str, Callable[[], object]]] = [ + ("upgrade", lambda: command.upgrade(config, "head")) + ] + if check: + steps.append(("check", lambda: command.check(config))) + steps.append(("downgrade", lambda: command.downgrade(config, "base"))) + steps.extend(steps[: 2 if check else 1]) + for name, step in steps: + failure = _step_failure(name, step) + if failure is not None: + return [failure] + return [] + + +def applied_revision(database_url: str) -> str | None: + """Return the revision the database at ``database_url`` has applied.""" + engine = create_simulation_engine(database_url) + try: + with engine.connect() as connection: + return read_schema_revision(connection) + finally: + engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/ownership_checks.py b/libs/policyengine-simulation-persistence/tests/ownership_checks.py new file mode 100644 index 000000000..6ae57384a --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/ownership_checks.py @@ -0,0 +1,273 @@ +"""Checks behind the simulation schema ownership rules in rules/registry.toml. + +Each check takes the files it inspects, so the rule tests run it on the +repository and the proof tests run it on the fixtures under +``rules/fixtures//``. Runtime code that changes a schema is caught by +sim.boundaries.sqlalchemy-and-alembic-only, not here. +""" + +from __future__ import annotations + +import ast +import configparser +import re +from pathlib import Path +from typing import TYPE_CHECKING + +import tomllib +import yaml + +if TYPE_CHECKING: + from collections.abc import Iterator, Mapping + +CHAIN_DIRECTORY = Path("migrations") +CHAIN_CONFIG = CHAIN_DIRECTORY / "alembic.ini" +CHAIN_REVISIONS = CHAIN_DIRECTORY / "versions" +SIMULATION_SCHEMA = "simulation" +IGNORED_DIRECTORY_NAMES = frozenset( + {".git", ".venv", "node_modules", "__pycache__", ".pytest_cache", ".ruff_cache"} +) +# Proof fixtures deliberately contain bad layouts. +IGNORED_RELATIVE_DIRECTORIES = (Path("rules") / "fixtures",) +PERSISTENCE_DISTRIBUTION = "policyengine-simulation-persistence" +PERSISTENCE_PACKAGE = "policyengine_simulation_persistence" +# Any variable named like a database URL (DATABASE_URL itself included), or a +# PostgreSQL URL written out. +DATABASE_CREDENTIAL = re.compile( + r"\b[A-Z0-9_]*DATABASE_URL[A-Z0-9_]*\b|\bpostgres(?:ql)?(?:\+\w+)?://" +) +# The comparison path's own database URL, and the name of the Secret Manager +# secret that holds it, may reach Modal until the comparison path is removed. +COMPARISON_PATH_CREDENTIALS = frozenset( + {"STAGE12_DATABASE_URL", "STAGE12_DATABASE_URL_SECRET_NAME"} +) +MODAL_SECRET_COMMAND = re.compile(r"\bmodal\s+secret\s+create\b") +# Deploying an application or running a temporary one both start Modal +# containers that a step's values may reach. +MODAL_DEPLOY_COMMAND = re.compile(r"\bmodal\s+(?:deploy|run)\b") + + +def _repository_files(root: Path, pattern: str) -> Iterator[Path]: + for path in sorted(root.rglob(pattern)): + relative = path.relative_to(root) + if IGNORED_DIRECTORY_NAMES.intersection(relative.parts): + continue + if any( + relative.is_relative_to(ignored) for ignored in IGNORED_RELATIVE_DIRECTORIES + ): + continue + yield path + + +def _configures_simulation_version_table(environment_source: str) -> bool: + for node in ast.walk(ast.parse(environment_source)): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "configure" + ): + continue + for keyword in node.keywords: + if keyword.arg != "version_table_schema": + continue + value = keyword.value + if isinstance(value, ast.Name) and value.id == "SIMULATION_SCHEMA": + return True + if isinstance(value, ast.Constant) and value.value == SIMULATION_SCHEMA: + return True + return False + + +def _layout_violations(root: Path) -> list[str]: + violations: list[str] = [] + configurations = [ + path.relative_to(root) for path in _repository_files(root, "alembic*.ini") + ] + if configurations != [CHAIN_CONFIG]: + violations.append( + f"expected only {CHAIN_CONFIG} as an Alembic configuration, found " + f"{[str(path) for path in configurations]}" + ) + for versions in _repository_files(root, "versions"): + relative = versions.relative_to(root) + if ( + versions.is_dir() + and relative != CHAIN_REVISIONS + and any(versions.glob("*.py")) + ): + violations.append(f"revision directory outside the chain: {relative}") + for environment in _repository_files(root, "env.py"): + relative = environment.relative_to(root) + if relative != CHAIN_DIRECTORY / "env.py" and "alembic" in ( + environment.read_text(encoding="utf-8") + ): + violations.append(f"Alembic environment outside the chain: {relative}") + return violations + + +def migration_chain_violations(root: Path) -> list[str]: + """Return how ``root`` departs from one simulation chain at ``migrations/``.""" + root = root.resolve() + violations = _layout_violations(root) + config_path = root / CHAIN_CONFIG + if not config_path.is_file(): + return violations + parser = configparser.ConfigParser(interpolation=None) + parser.read(config_path, encoding="utf-8") + script_location = parser.get("alembic", "script_location", fallback="") + resolved = Path(script_location.replace("%(here)s", str(config_path.parent))) + if not resolved.is_absolute(): + resolved = root / resolved + if resolved.resolve() != (root / CHAIN_DIRECTORY).resolve(): + violations.append(f"script_location {script_location!r} is not migrations/") + environment_path = root / CHAIN_DIRECTORY / "env.py" + if not environment_path.is_file() or not _configures_simulation_version_table( + environment_path.read_text(encoding="utf-8") + ): + violations.append( + "migrations/env.py must keep its version table in the simulation schema" + ) + return violations + + +def declares_persistence_dependency(pyproject_text: str) -> bool: + """Return whether a pyproject declares the persistence library anywhere.""" + project = tomllib.loads(pyproject_text).get("project", {}) + requirements = [ + *project.get("dependencies", []), + *( + requirement + for group in project.get("optional-dependencies", {}).values() + for requirement in group + ), + ] + return any( + re.split(r"[\s\[<>=!~;]", requirement, maxsplit=1)[0] + == PERSISTENCE_DISTRIBUTION + for requirement in requirements + ) + + +def persistence_dependency_violations(project_root: Path) -> list[str]: + """Return where a project depends on or imports the persistence library.""" + violations: list[str] = [] + pyproject = project_root / "pyproject.toml" + if declares_persistence_dependency(pyproject.read_text(encoding="utf-8")): + violations.append(f"{pyproject} declares {PERSISTENCE_DISTRIBUTION}") + lock = project_root / "uv.lock" + if lock.is_file() and f'name = "{PERSISTENCE_DISTRIBUTION}"' in lock.read_text( + encoding="utf-8" + ): + violations.append(f"{lock} locks {PERSISTENCE_DISTRIBUTION}") + violations.extend( + f"{path} imports {PERSISTENCE_PACKAGE}" + for path in sorted((project_root / "src").rglob("*.py")) + if PERSISTENCE_PACKAGE in path.read_text(encoding="utf-8") + ) + return violations + + +def _foreign_credentials(text: str) -> list[str]: + """Return the database credentials in ``text`` other than the comparison path's.""" + return sorted( + { + match + for match in DATABASE_CREDENTIAL.findall(text) + if match not in COMPARISON_PATH_CREDENTIALS + } + ) + + +def _environment_text(environment: object) -> str: + if not isinstance(environment, dict): + return "" + return "\n".join(f"{key}={value}" for key, value in environment.items()) + + +def _step_text(step: Mapping[str, object]) -> str: + return "\n".join( + str(value) + for value in (step.get("run", ""), step.get("uses", ""), step.get("with", "")) + ) + + +def _step_reaches_modal(step_text: str, modal_scripts: set[str]) -> bool: + return bool( + MODAL_SECRET_COMMAND.search(step_text) + or MODAL_DEPLOY_COMMAND.search(step_text) + or any(name in step_text for name in modal_scripts) + ) + + +def _workflow_violations( + workflow_name: str, workflow_text: str, modal_scripts: set[str] +) -> list[str]: + workflow = yaml.safe_load(workflow_text) or {} + workflow_environment = _environment_text(workflow.get("env")) + violations: list[str] = [] + for job_name, job in (workflow.get("jobs") or {}).items(): + job_environment = _environment_text(job.get("env")) + for step in job.get("steps") or []: + step_text = _step_text(step) + if not _step_reaches_modal(step_text, modal_scripts): + continue + # Workflow-level and job-level env reach every step of the job. + carried = _foreign_credentials( + "\n".join( + ( + workflow_environment, + job_environment, + _environment_text(step.get("env")), + step_text, + ) + ) + ) + if carried: + violations.append( + f"{workflow_name} job {job_name} step " + f"{step.get('name', '')!r} gives Modal a database " + f"credential other than the comparison path's: " + f"{', '.join(carried)}" + ) + return violations + + +def modal_credential_violations( + workflow_texts: Mapping[str, str], + script_texts: Mapping[str, str], +) -> list[str]: + """Return Modal secret or deploy steps that carry a foreign database URL. + + ``workflow_texts`` and ``script_texts`` map a file name to its contents. A + step counts as giving Modal a value when it creates a Modal secret, deploys + a Modal application or runs a temporary one, directly or through a script + that does, and + every value in its step, job or workflow environment reaches it. Only the + comparison path's own database URL may reach Modal. + """ + modal_scripts = { + name + for name, text in script_texts.items() + if MODAL_SECRET_COMMAND.search(text) or MODAL_DEPLOY_COMMAND.search(text) + } + violations = [ + f"{name} gives Modal a database credential other than the comparison " + f"path's: {', '.join(carried)}" + for name in sorted(modal_scripts) + if (carried := _foreign_credentials(script_texts[name])) + ] + for workflow_name, workflow_text in sorted(workflow_texts.items()): + violations.extend( + _workflow_violations(workflow_name, workflow_text, modal_scripts) + ) + return violations + + +def modal_source_violations(sources: Mapping[str, str]) -> list[str]: + """Return Modal project sources that name a foreign database URL variable.""" + return [ + f"{name} names {variable}" + for name, source in sorted(sources.items()) + for variable in _foreign_credentials(source) + if not variable.endswith("://") + ] diff --git a/libs/policyengine-simulation-persistence/tests/test_migration_cycle.py b/libs/policyengine-simulation-persistence/tests/test_migration_cycle.py new file mode 100644 index 000000000..d033ec4aa --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_migration_cycle.py @@ -0,0 +1,198 @@ +"""Run the simulation migration chain against a fresh PostgreSQL database.""" + +from __future__ import annotations + +import time + +import pytest +from alembic import command +from migration_support import ( + MIGRATOR_ROLE, + REPOSITORY_ROOT, + alembic_config, + applied_revision, + chain_config, + migration_cycle_failures, +) +from policyengine_simulation_persistence import ( + HEAD_REVISION, + SIMULATION_SCHEMA, + create_simulation_engine, +) +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.exc import OperationalError, ProgrammingError + +pytestmark = pytest.mark.postgres + + +def _superuser_engine(database, **options): + return create_engine( + database.superuser_url.replace("postgresql://", "postgresql+psycopg://", 1), + **options, + ) + + +def test_upgrade_check_downgrade_upgrade_cycle(disposable_database) -> None: + config = alembic_config() + + assert migration_cycle_failures(config, check=True) == [] + assert applied_revision(disposable_database.migrator_url) == HEAD_REVISION + + command.downgrade(config, "base") + assert applied_revision(disposable_database.migrator_url) is None + + +def test_rule_proof_migration_cycle(disposable_database) -> None: + """Proof for rule sim.persistence.migration-cycle (rules/registry.toml).""" + fixtures = ( + REPOSITORY_ROOT / "rules" / "fixtures" / "sim.persistence.migration-cycle" + ) + + assert ( + migration_cycle_failures(chain_config(fixtures / "good_chain"), check=False) + == [] + ) + assert migration_cycle_failures( + chain_config(fixtures / "bad_chain"), check=False + ) == [ + "upgrade failed: ProgrammingError: (psycopg.errors.DuplicateTable) relation " + '"proof_bad_table" already exists' + ] + + +def test_the_version_table_lives_only_in_the_simulation_schema( + disposable_database, +) -> None: + command.upgrade(alembic_config(), "head") + + engine = _superuser_engine(disposable_database) + try: + with engine.connect() as connection: + inspector = inspect(connection) + assert inspector.has_table("alembic_version", schema=SIMULATION_SCHEMA) + assert not inspector.has_table("alembic_version", schema="public") + schema_owner = connection.execute( + text( + "SELECT pg_get_userbyid(nspowner) FROM pg_namespace " + "WHERE nspname = :schema" + ), + {"schema": SIMULATION_SCHEMA}, + ).scalar() + finally: + engine.dispose() + assert schema_owner == MIGRATOR_ROLE + + +def test_an_unmigrated_database_reports_no_revision(disposable_database) -> None: + assert applied_revision(disposable_database.migrator_url) is None + + +def _precreate_schema_without_database_create(database) -> None: + """Set up the operator alternative to granting the migrator CREATE. + + The superuser takes over the database, so the migrator keeps only the + default CONNECT and TEMPORARY privileges, and creates the simulation + schema owned by the migrator. + """ + engine = _superuser_engine(database, isolation_level="AUTOCOMMIT") + try: + with engine.connect() as connection: + connection.execute( + text(f'ALTER DATABASE "{database.name}" OWNER TO CURRENT_USER') + ) + connection.execute( + text(f"CREATE SCHEMA {SIMULATION_SCHEMA} AUTHORIZATION {MIGRATOR_ROLE}") + ) + migrator_may_create = connection.execute( + text( + "SELECT has_database_privilege(:role, current_database(), 'CREATE')" + ), + {"role": MIGRATOR_ROLE}, + ).scalar() + finally: + engine.dispose() + assert migrator_may_create is False + + +def test_a_precreated_schema_needs_no_database_create_privilege( + disposable_database, +) -> None: + _precreate_schema_without_database_create(disposable_database) + + command.upgrade(alembic_config(), "head") + + assert applied_revision(disposable_database.migrator_url) == HEAD_REVISION + assert applied_revision(disposable_database.runtime_url) == HEAD_REVISION + + +def test_the_runtime_role_reads_the_revision_but_cannot_change_the_schema( + disposable_database, +) -> None: + command.upgrade(alembic_config(), "head") + + assert applied_revision(disposable_database.runtime_url) == HEAD_REVISION + engine = create_simulation_engine(disposable_database.runtime_url) + try: + with ( + pytest.raises(ProgrammingError, match="permission denied"), + engine.begin() as connection, + ): + connection.execute( + text("CREATE TABLE simulation.unowned (identifier integer)") + ) + with ( + pytest.raises(ProgrammingError, match="permission denied"), + engine.begin() as connection, + ): + connection.execute( + text("UPDATE simulation.alembic_version SET version_num = ''") + ) + finally: + engine.dispose() + + +def test_a_failed_revision_leaves_nothing_applied( + disposable_database, monkeypatch +) -> None: + """The schema and every revision apply in one transaction.""" + from alembic.runtime import migration + + def fail_after_creating_the_schema(*_arguments, **_keywords): + msg = "simulated failure inside the migration transaction" + raise RuntimeError(msg) + + monkeypatch.setattr( + migration.MigrationContext, + "run_migrations", + fail_after_creating_the_schema, + ) + + with pytest.raises(RuntimeError, match="simulated failure"): + command.upgrade(alembic_config(), "head") + + engine = _superuser_engine(disposable_database) + try: + with engine.connect() as connection: + assert not inspect(connection).has_schema(SIMULATION_SCHEMA) + finally: + engine.dispose() + + +def test_a_migration_waiting_for_a_lock_gives_up_instead_of_blocking( + disposable_database, +) -> None: + """Writers queue behind a waiting migration, so it must not wait long.""" + command.upgrade(alembic_config(), "head") + holder = _superuser_engine(disposable_database) + try: + with holder.begin() as connection: + connection.execute( + text("LOCK TABLE simulation.alembic_version IN ACCESS EXCLUSIVE MODE") + ) + started = time.monotonic() + with pytest.raises(OperationalError, match="lock timeout"): + command.upgrade(alembic_config(), "head") + waited = time.monotonic() - started + finally: + holder.dispose() + assert waited < 30 diff --git a/libs/policyengine-simulation-persistence/tests/test_migration_script.py b/libs/policyengine-simulation-persistence/tests/test_migration_script.py new file mode 100644 index 000000000..e7b09e217 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_migration_script.py @@ -0,0 +1,54 @@ +"""The deploy job's migration script, run for real against PostgreSQL.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest +from migration_support import ( + MIGRATION_DATABASE_URL_VARIABLE, + REPOSITORY_ROOT, + applied_revision, +) +from policyengine_simulation_persistence import ( + HEAD_REVISION, +) + +pytestmark = pytest.mark.postgres + +SCRIPT = REPOSITORY_ROOT / ".github" / "scripts" / "migrate-simulation-database.sh" +LIBRARY_DIRECTORY = Path(__file__).resolve().parents[1] + + +def _run_script(database, mode: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(SCRIPT)], + cwd=LIBRARY_DIRECTORY, + env={ + **os.environ, + MIGRATION_DATABASE_URL_VARIABLE: database.migrator_url, + "SIMULATION_MIGRATION_MODE": mode, + }, + capture_output=True, + text=True, + check=False, + ) + + +def test_verify_mode_refuses_a_database_behind_the_chain(disposable_database) -> None: + result = _run_script(disposable_database, "verify") + + assert result.returncode != 0 + assert applied_revision(disposable_database.migrator_url) is None + + +def test_upgrade_mode_migrates_and_then_verify_mode_passes(disposable_database) -> None: + upgraded = _run_script(disposable_database, "upgrade") + assert upgraded.returncode == 0, upgraded.stderr + assert applied_revision(disposable_database.migrator_url) == HEAD_REVISION + + verified = _run_script(disposable_database, "verify") + assert verified.returncode == 0, verified.stderr + assert "simulation-test-password" not in upgraded.stdout + upgraded.stderr diff --git a/libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py b/libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py new file mode 100644 index 000000000..eb66ec159 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py @@ -0,0 +1,178 @@ +"""Simulation schema ownership: one chain, no executor dependency, none in Modal.""" + +from __future__ import annotations + +import pytest +from migration_support import REPOSITORY_ROOT +from ownership_checks import ( + declares_persistence_dependency, + migration_chain_violations, + modal_credential_violations, + modal_source_violations, + persistence_dependency_violations, +) + +FIXTURES = REPOSITORY_ROOT / "rules" / "fixtures" +WORKFLOWS = REPOSITORY_ROOT / ".github" / "workflows" +WORKFLOW_SCRIPTS = REPOSITORY_ROOT / ".github" / "scripts" +MODAL_PROJECTS = ( + REPOSITORY_ROOT / "projects" / "policyengine-simulation-executor", + REPOSITORY_ROOT / "projects" / "policyengine-simulation-gateway", +) + + +def _texts(directory, pattern): + return { + path.name: path.read_text(encoding="utf-8") + for path in sorted(directory.glob(pattern)) + } + + +def test_repository_has_one_simulation_migration_chain() -> None: + assert migration_chain_violations(REPOSITORY_ROOT) == [] + + +def test_rule_proof_one_migration_chain() -> None: + """Proof for rule sim.persistence.one-migration-chain (rules/registry.toml).""" + fixtures = FIXTURES / "sim.persistence.one-migration-chain" + + assert migration_chain_violations(fixtures / "bad_repository") != [] + assert migration_chain_violations(fixtures / "good_repository") == [] + + +def test_modal_projects_never_depend_on_the_persistence_library() -> None: + for project_root in MODAL_PROJECTS: + assert persistence_dependency_violations(project_root) == [] + + +def test_rule_proof_executor_independent() -> None: + """Proof for rule sim.persistence.executor-independent (rules/registry.toml).""" + fixtures = FIXTURES / "sim.persistence.executor-independent" + + assert declares_persistence_dependency( + (fixtures / "bad_pyproject.toml").read_text(encoding="utf-8") + ) + assert not declares_persistence_dependency( + (fixtures / "good_pyproject.toml").read_text(encoding="utf-8") + ) + + +def test_modal_receives_no_database_url_but_the_comparison_paths() -> None: + assert ( + modal_credential_violations( + _texts(WORKFLOWS, "*.yml"), + _texts(WORKFLOW_SCRIPTS, "*.sh"), + ) + == [] + ) + for project_root in MODAL_PROJECTS: + sources = { + str(path): path.read_text(encoding="utf-8") + for path in sorted((project_root / "src").rglob("*.py")) + } + assert modal_source_violations(sources) == [] + + +def test_rule_proof_no_modal_database_credentials() -> None: + """Proof for rule sim.persistence.no-modal-database-credentials.""" + fixtures = FIXTURES / "sim.persistence.no-modal-database-credentials" + no_scripts: dict[str, str] = {} + + assert modal_credential_violations( + {"bad.yml": (fixtures / "bad_workflow.yml").read_text(encoding="utf-8")}, + no_scripts, + ) + assert not modal_credential_violations( + {"good.yml": (fixtures / "good_workflow.yml").read_text(encoding="utf-8")}, + no_scripts, + ) + + +def _modal_secret_workflow(*, step_env: str = "", job_env: str = "") -> str: + return ( + "jobs:\n" + " deploy:\n" + f"{job_env}" + " steps:\n" + " - name: Synchronize Modal secrets\n" + f"{step_env}" + " run: uv run modal secret create runtime --from-dotenv .env\n" + ) + + +@pytest.mark.parametrize( + ("step_env", "job_env"), + [ + (" env:\n REPORTS_DATABASE_URL: x\n", ""), + (" env:\n DATABASE_URL: x\n", ""), + (" env:\n SIMULATION_DATABASE_URL: x\n", ""), + ("", " env:\n SIMULATION_MIGRATION_DATABASE_URL: x\n"), + (" env:\n DSN: postgresql://user:secret@db/postgres\n", ""), + ], + ids=[ + "any-database-url", + "bare-database-url", + "simulation-url", + "job-level-env", + "url-literal", + ], +) +def test_modal_steps_may_carry_only_the_comparison_paths_database_url( + step_env, job_env +) -> None: + workflow = _modal_secret_workflow(step_env=step_env, job_env=job_env) + + assert modal_credential_violations({"deploy.yml": workflow}, {}) != [] + + +def test_the_comparison_paths_database_url_may_reach_modal_until_it_is_removed() -> ( + None +): + workflow = _modal_secret_workflow( + step_env=( + " env:\n" + " STAGE12_DATABASE_URL_SECRET_NAME: stage12-database-url\n" + ) + ) + script = "jq -n '{STAGE12_DATABASE_URL: $url}'\nuv run modal secret create x\n" + + assert modal_credential_violations({"deploy.yml": workflow}, {}) == [] + assert modal_credential_violations({}, {"stage12-sync.sh": script}) == [] + + +def test_a_modal_script_may_not_carry_another_database_url() -> None: + script = ( + "uv run modal secret create x REPORTS_DATABASE_URL=${REPORTS_DATABASE_URL}\n" + ) + + assert modal_credential_violations({}, {"sync.sh": script}) == [ + "sync.sh gives Modal a database credential other than the comparison " + "path's: REPORTS_DATABASE_URL" + ] + + +def test_modal_sources_may_name_only_the_comparison_paths_database_url() -> None: + assert ( + modal_source_violations({"worker.py": "os.environ['STAGE12_DATABASE_URL']"}) + == [] + ) + assert modal_source_violations( + {"worker.py": "os.environ['SIMULATION_DATABASE_URL']"} + ) == ["worker.py names SIMULATION_DATABASE_URL"] + assert modal_source_violations({"worker.py": "os.getenv('DATABASE_URL')"}) == [ + "worker.py names DATABASE_URL" + ] + + +def test_a_temporary_modal_app_may_not_receive_another_database_url() -> None: + workflow = ( + "jobs:\n" + " precompute:\n" + " steps:\n" + " - name: Precompute\n" + " env:\n" + " SIMULATION_DATABASE_URL: x\n" + " run: uv run --frozen modal run src/modal/precompute_app.py\n" + ) + + assert modal_credential_violations({"precompute.yml": workflow}, {}) != [] diff --git a/libs/policyengine-simulation-persistence/tests/test_schema.py b/libs/policyengine-simulation-persistence/tests/test_schema.py new file mode 100644 index 000000000..d42b8bf4d --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_schema.py @@ -0,0 +1,106 @@ +"""Schema names and connection settings, without a database.""" + +from __future__ import annotations + +import pytest +from alembic import command +from alembic.script import ScriptDirectory +from migration_support import MIGRATION_DATABASE_URL_VARIABLE, alembic_config +from policyengine_simulation_persistence import ( + HEAD_REVISION, + RUNTIME_ROLE, + SIMULATION_SCHEMA, + create_simulation_engine, + normalized_database_url, + simulation_metadata, +) + + +def test_the_schema_and_runtime_role_have_their_deployed_names() -> None: + assert SIMULATION_SCHEMA == "simulation" + assert RUNTIME_ROLE == "simulation_runtime" + + +def test_every_table_is_in_the_simulation_schema() -> None: + assert simulation_metadata.schema == SIMULATION_SCHEMA + for table in simulation_metadata.tables.values(): + assert table.schema == SIMULATION_SCHEMA, table.name + + +def test_database_url_selects_psycopg_without_changing_the_secret_format() -> None: + url = normalized_database_url( + "postgresql://simulation_runtime:secret@db.example:5432/postgres" + ) + + assert url.drivername == "postgresql+psycopg" + assert url.username == "simulation_runtime" + assert url.host == "db.example" + assert url.database == "postgres" + + +@pytest.mark.parametrize( + "database_url", + [ + "", + " postgresql://user:secret@db.example/postgres", + "postgres://user:secret@db.example/postgres", + "postgresql+psycopg://user:secret@db.example/postgres", + "postgresql://db.example/postgres", + "postgresql://user:secret@db.example", + "not a url", + ], +) +def test_malformed_database_urls_are_rejected_without_echoing_them( + database_url, +) -> None: + with pytest.raises(ValueError, match="Simulation database URL") as error: + normalized_database_url(database_url) + + assert "secret" not in str(error.value) + assert error.value.__cause__ is None or "secret" not in str(error.value.__cause__) + + +def test_runtime_engine_never_prepares_server_side_statements(monkeypatch) -> None: + captured_options = {} + + def capture_engine(url, **options): + captured_options.update(options, url=url) + return object() + + monkeypatch.setattr( + "policyengine_simulation_persistence.engine.create_engine", + capture_engine, + ) + + create_simulation_engine("postgresql://runtime:secret@db.example/postgres") + + assert captured_options["url"].drivername == "postgresql+psycopg" + assert captured_options["connect_args"]["prepare_threshold"] is None + assert captured_options["connect_args"]["connect_timeout"] > 0 + assert captured_options["pool_pre_ping"] is True + assert captured_options["pool_size"] >= 1 + + +def test_head_revision_names_the_single_head_of_the_chain() -> None: + script = ScriptDirectory.from_config(alembic_config()) + + assert script.get_heads() == [HEAD_REVISION] + + +def test_migrations_refuse_to_run_without_the_migration_database_url( + monkeypatch, +) -> None: + monkeypatch.delenv(MIGRATION_DATABASE_URL_VARIABLE, raising=False) + + with pytest.raises(RuntimeError, match=MIGRATION_DATABASE_URL_VARIABLE): + command.upgrade(alembic_config(), "head") + + +def test_migrations_refuse_offline_sql_generation(monkeypatch) -> None: + monkeypatch.setenv( + MIGRATION_DATABASE_URL_VARIABLE, + "postgresql://simulation_migrator:secret@db.example/postgres", + ) + + with pytest.raises(RuntimeError, match="online"): + command.upgrade(alembic_config(), "head", sql=True) diff --git a/libs/policyengine-simulation-persistence/uv.lock b/libs/policyengine-simulation-persistence/uv.lock new file mode 100644 index 000000000..b007a60ff --- /dev/null +++ b/libs/policyengine-simulation-persistence/uv.lock @@ -0,0 +1,307 @@ +version = 1 +revision = 5 +requires-python = "==3.13.*" + +[[package]] +name = "alembic" +version = "1.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/aa/02910bdb8e2f1444f6654d5b296cd827d126f82209050ee7b1000f92ac4b/alembic-1.20.0.tar.gz", hash = "sha256:db505480647bc60386c5369402f4a57a506b7539c9e9ef5e270d45cbbe4939bf", size = 2093272, upload-time = "2026-09-11T19:09:11.126Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/27/78a89b55b0904d222183164e079b4ca56208e94eff1d35ad1f1ad5be9b06/alembic-1.20.0-py3-none-any.whl", hash = "sha256:77eb101048d95f982c0353e9233404889dcd7a6fc244c107836c0e2fc9cf7d9d", size = 268719, upload-time = "2026-09-11T19:09:12.88Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/55/d1eaf3e73781174340a00dc1ba2aee8a65f82fadb18e2797b192b6b3925b/coverage-7.16.2.tar.gz", hash = "sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa", size = 971999, upload-time = "2026-09-27T12:29:01.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/f6/8eb4f220ef24f84fb27d852d4f9bf83e0c73ec1a4a08dd9a87e3f4529739/coverage-7.16.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1a37c6e478cf687e1aa30a593d19c92c02fad9d122b51ab73f51b8dc7a0c0fc9", size = 223993, upload-time = "2026-09-27T12:26:40.164Z" }, + { url = "https://files.pythonhosted.org/packages/40/23/d4bbaf0c154e0b0c2b5264890dbf6ef098dcb50ec8f2469be9490d191660/coverage-7.16.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0993d0e90858c03943d3cb152e068a20dd4707924deec84dd2230261baae3b1b", size = 224368, upload-time = "2026-09-27T12:26:41.762Z" }, + { url = "https://files.pythonhosted.org/packages/7f/48/fc1e88fd571ec5cb38150b7f89f7696ca1bdf9920e01432febb69774cc85/coverage-7.16.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204", size = 255358, upload-time = "2026-09-27T12:26:43.442Z" }, + { url = "https://files.pythonhosted.org/packages/1d/56/6785397d07c29c8e70fbb9a07e97d062b43c21ffc5f12385917847f09f63/coverage-7.16.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5", size = 257955, upload-time = "2026-09-27T12:26:45.725Z" }, + { url = "https://files.pythonhosted.org/packages/27/3b/c8cdd07721e5f99abd81cea970d971997f99bf158c0b85f51bd284179c8b/coverage-7.16.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7", size = 259183, upload-time = "2026-09-27T12:26:47.208Z" }, + { url = "https://files.pythonhosted.org/packages/9b/11/606b192fe43d32574ec6238549d48de588fdcc18485682a5ec0a8ac357f2/coverage-7.16.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541", size = 261325, upload-time = "2026-09-27T12:26:49.084Z" }, + { url = "https://files.pythonhosted.org/packages/67/90/eea481f8b0305ceeb33f081a5f47e298391dbd1b589de0c4b3b3aa50d3f2/coverage-7.16.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc", size = 255531, upload-time = "2026-09-27T12:26:50.509Z" }, + { url = "https://files.pythonhosted.org/packages/6b/be/dedbf9aea1457b120c27ac10b8fc2a357f37fa2b54c3e7286d42980a0a2a/coverage-7.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967", size = 257329, upload-time = "2026-09-27T12:26:52.005Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cb/b25c19d5bb2bd0f2e4e27fe8e2ffcae80c7a91ae181c0dc749ed60e9b1a4/coverage-7.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7", size = 255294, upload-time = "2026-09-27T12:26:53.634Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a2/892c5c5f4ad44b7b2ca009aee705191f3f268f15052244f2f9e3539b2e35/coverage-7.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23", size = 259444, upload-time = "2026-09-27T12:26:55.243Z" }, + { url = "https://files.pythonhosted.org/packages/ed/99/a562537deba0a3e370182ae71c149be796c39d8087365f17a09188f27145/coverage-7.16.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8", size = 255111, upload-time = "2026-09-27T12:26:56.851Z" }, + { url = "https://files.pythonhosted.org/packages/2d/20/854ec68641a9b3362ff068a32dfa41637299761617ef253791dbade6fc76/coverage-7.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4", size = 256882, upload-time = "2026-09-27T12:26:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/db/0d/748e4518b0ac0f9ff2687c248a6e5f8c0737306e709372632a2556f84443/coverage-7.16.2-cp313-cp313-win32.whl", hash = "sha256:f80bd9f9633eafc73d0a913ba2645c96ba58bba1befc30590f7c0fbfde59d865", size = 226045, upload-time = "2026-09-27T12:26:59.983Z" }, + { url = "https://files.pythonhosted.org/packages/31/fa/6e46edba66a183fe4d99d4bb52c173287e9b8dddabe0888d24cb8210e580/coverage-7.16.2-cp313-cp313-win_amd64.whl", hash = "sha256:8be099e979fc42559328a21828281b4578304191ae46ed4e80a407048a82eee6", size = 226581, upload-time = "2026-09-27T12:27:01.494Z" }, + { url = "https://files.pythonhosted.org/packages/1b/d9/9ef6845367600b336ff75d000444a0d32497d6972c833141bd39356abf68/coverage-7.16.2-cp313-cp313-win_arm64.whl", hash = "sha256:28ff850182a67d117990fa2ce5ea1032836d8c9630dae867e8bdd3bff4533b79", size = 226409, upload-time = "2026-09-27T12:27:03.116Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0c/7a64e1ac90541a8edf50daef0914848011fb057a5bf55284a4811e21939a/coverage-7.16.2-py3-none-any.whl", hash = "sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f", size = 215754, upload-time = "2026-09-27T12:28:59.075Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "mako" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/09/e07c4b5579a79f4b16f8d4f29f6c54514ac787c4ad506b8c4f28a0e6b0bf/mako-1.4.3.tar.gz", hash = "sha256:cd6537fe88d5fec315c55c2f8529bc4ce7a9a352ad7db3eeaa6a66e2dd4ec37a", size = 412799, upload-time = "2026-09-22T20:54:31.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/a0/053d6af3e8f871e0073b4a36732d9e65be77a72e5434c31b94f6af78a6bb/mako-1.4.3-py3-none-any.whl", hash = "sha256:723296007c870bfd6b3f0c3230dba7198096e5269297ebf5e4eff9e7ffa39d4f", size = 80164, upload-time = "2026-09-22T20:54:33.128Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/18/4bc5ba32499e87bb2b0ef5b3a9bb9c00a131fa961ddf0be548cb550f548b/markupsafe-3.0.4-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1", size = 12786, upload-time = "2026-10-02T23:05:08.709Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6f/17f0c099bf25f3e31e63cc19244d9f6af861a9a4ab778c203997903cfdd0/markupsafe-3.0.4-cp313-cp313-android_24_x86_64.whl", hash = "sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1", size = 12510, upload-time = "2026-10-02T23:05:09.93Z" }, + { url = "https://files.pythonhosted.org/packages/11/af/1a141081b905036ee904ec4bd945e1f70b4e1b32d33c4e59e8cf1d58b247/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96", size = 11583, upload-time = "2026-10-02T23:05:10.884Z" }, + { url = "https://files.pythonhosted.org/packages/e7/0a/a89385ae590232622a03e091805cff12f24fabe6c11e0e8bae096cece81c/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148", size = 11801, upload-time = "2026-10-02T23:05:11.913Z" }, + { url = "https://files.pythonhosted.org/packages/ed/85/ea548dc013962eb73653124bc595635fbf9e0fa41d1f181a967ccb784dfb/markupsafe-3.0.4-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e", size = 11443, upload-time = "2026-10-02T23:05:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/cc/72/15f2e5ec9cf2eb00d5cdfe968d94e4156a7bd7303832c3f3b2c403a36839/markupsafe-3.0.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248", size = 11665, upload-time = "2026-10-02T23:05:13.829Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e0/4030bea613677e333c8a2c901fd405055f657f9d06acba5b7357984b6ef7/markupsafe-3.0.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72", size = 12061, upload-time = "2026-10-02T23:05:14.807Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a5/28b76a7449eb702966b88bef599e2360b411fbb3afeee8fe560939be06ec/markupsafe-3.0.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2", size = 24395, upload-time = "2026-10-02T23:05:15.909Z" }, + { url = "https://files.pythonhosted.org/packages/07/6c/21232811afc3a063b5e934b1ae2efda52f46154ec382f585149c020e61fe/markupsafe-3.0.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85", size = 27183, upload-time = "2026-10-02T23:05:16.976Z" }, + { url = "https://files.pythonhosted.org/packages/14/38/6ccdfa5b59049cb36fb80cbc80aee9cf1fc9bb77d1335ad435f2070b08cf/markupsafe-3.0.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde", size = 24772, upload-time = "2026-10-02T23:05:18.209Z" }, + { url = "https://files.pythonhosted.org/packages/63/e0/cec6865dfe88cb48fedd4b20aed6af5158e41092adcbf3e028bcc6ec2108/markupsafe-3.0.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6", size = 23017, upload-time = "2026-10-02T23:05:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/ee/76/6ed4940bb7648a9aac457c14f870cfdd5105f139a0fb1f29cd61fafa47d1/markupsafe-3.0.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f", size = 22036, upload-time = "2026-10-02T23:05:20.352Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4f/ed476226d4fe46a09090a36025bf319296810028df55eb12f1253b540f3a/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39", size = 23786, upload-time = "2026-10-02T23:05:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/9a/35/66ff30450e35ef5fba9ebc930c9411747e537fd9447b65e44f5007e2b84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee", size = 22672, upload-time = "2026-10-02T23:05:22.922Z" }, + { url = "https://files.pythonhosted.org/packages/32/0b/72f45ce4b4efcbca4b80cf1b06703eff0be8d37e82abb78f66c85a7ead1e/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2", size = 24107, upload-time = "2026-10-02T23:05:24.175Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/71776e5fdcba04614b384cc102e8a4198208579d896fd1394cb7cb9aa900/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46", size = 21615, upload-time = "2026-10-02T23:05:25.215Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/801ce02a02e7aee0f784b1ec7843026178f6adeb9c93ac67eb1992a9a84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17", size = 23076, upload-time = "2026-10-02T23:05:26.423Z" }, + { url = "https://files.pythonhosted.org/packages/4a/85/c43776625428f3bb4a61e8633940400e3efe6409e3c6f5bff26de5e45618/markupsafe-3.0.4-cp313-cp313-win32.whl", hash = "sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0", size = 14118, upload-time = "2026-10-02T23:05:27.716Z" }, + { url = "https://files.pythonhosted.org/packages/6f/36/163da64de88a13db79214ef75fa041be7fa13bdb42261cf5b7484de14bfb/markupsafe-3.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5", size = 14313, upload-time = "2026-10-02T23:05:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a8/9b662783ffaa1149221432a923cee562f78b9cbbb8baa3df9b3753e63e1e/markupsafe-3.0.4-cp313-cp313-win_arm64.whl", hash = "sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc", size = 14189, upload-time = "2026-10-02T23:05:29.917Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/105de02c1322cfada6d9710d9146ef8026419d433c9d08359a2d35805811/nodeenv-1.11.0.tar.gz", hash = "sha256:3ce8fe5b71d16e8af7039ca65257354100bc772965d6bc549070649e53b1b146", size = 99301, upload-time = "2026-09-26T11:29:21.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/c8/12811c9b48fde162bb72b6f2e78fada9a247a09d7bb5be2050a5d099c77b/nodeenv-1.11.0-py2.py3-none-any.whl", hash = "sha256:edaa16e6c14d7cf395d75d4bbd5a26390f4dc06501a33b4e76282b02cc688a25", size = 34277, upload-time = "2026-09-26T11:29:19.933Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.optional-dependencies] +build = [ + { name = "pyright" }, +] +migrations = [ + { name = "alembic" }, +] +test = [ + { name = "alembic" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", marker = "extra == 'migrations'", specifier = ">=1.17.1,<2" }, + { name = "alembic", marker = "extra == 'test'", specifier = ">=1.17.1,<2" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<3" }, +] +provides-extras = ["migrations", "test", "build"] + +[[package]] +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/c3/c072584b69ad44a747b448cfc9766fecb8aae56e372a017e2ef668790057/psycopg_binary-3.3.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6", size = 4712284, upload-time = "2026-09-18T13:19:13.451Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b9/4283b785339e8e2318d03048994b093d650ea6289fabaa806b765dc0d449/psycopg_binary-3.3.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f", size = 4772031, upload-time = "2026-09-18T13:19:18.524Z" }, + { url = "https://files.pythonhosted.org/packages/6f/72/7a1321d359246769fff1affffbd0132785a28f7f63c18524c15a502398f4/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9", size = 5556392, upload-time = "2026-09-18T13:19:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/de/b0/c6f8a0585a5dacbea74e130bcfc66629390e8f5bbc79d2a8e806e8952150/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269", size = 5237855, upload-time = "2026-09-18T13:19:31.257Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fc/c3a7a8bbef7e945ec584ac61d460a612363ea398511cd0e220242b1d69f1/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef", size = 6833856, upload-time = "2026-09-18T13:19:43.622Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f2/8e80b921db728ebb68fc105bd7c4277f908210ad755bd6481d5ea7add740/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784", size = 5070730, upload-time = "2026-09-18T13:19:49.968Z" }, + { url = "https://files.pythonhosted.org/packages/54/6a/5b313e0c5348244f0e973aff3258bf86766656256d5ece8d541a53e35b4a/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc", size = 4598089, upload-time = "2026-09-18T13:19:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/32/e9/db7f76ec24bf6699e92bf604e5c4bae10664a681a8999ef42aa0faf0f2c6/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8", size = 4278481, upload-time = "2026-09-18T13:20:04.681Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/72c67013656f4d6b547caabffb193e91d57e63f90eefdcc6d045c400e97d/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22", size = 4009229, upload-time = "2026-09-18T13:20:11.905Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/5e4500df2c999eb0faed8b184e6958b834172128274f06167a5deef4c19c/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138", size = 4321467, upload-time = "2026-09-18T13:20:17.949Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/e350e1cf498ba2565c3f87b12f429d2012eb86b76c2b3845a19ee5fbb4d6/psycopg_binary-3.3.6-cp313-cp313-win_amd64.whl", hash = "sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372", size = 3658179, upload-time = "2026-09-18T13:20:22.691Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pyright" +version = "1.1.414" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/1b/244c7b710031ada80f27e579ec20d28a2285dfc318fed0339866b1047f12/pyright-1.1.414.tar.gz", hash = "sha256:523c0a97c60da6333234955c277730c9cf4f5bd6d5399e7b7d2b0fc5d3599524", size = 4154638, upload-time = "2026-09-10T12:26:53.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/ba/18b6e682ead424ad24bcc134339ae5d1b931cd9ae260540592a058a91279/pyright-1.1.414-py3-none-any.whl", hash = "sha256:2a6b4b3298c9eec174c5ed83bd338de6eee82df2992f3e1930e6199d381be36f", size = 6225049, upload-time = "2026-09-10T12:26:51.427Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/dc/9addbf490110b5727380059ee542ad8353bdcee45a37ba6a5e0976966ae0/sqlalchemy-2.1.6.tar.gz", hash = "sha256:499eae5c9051efe96cb9a73ec2fbde8ffa4186f102b9094ebab8fcc9d3b99f52", size = 10589700, upload-time = "2026-10-21T16:25:06.497Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/db/ecf1cf1f5c8cb7661081dcb0561633c174213d7af5fccc17ab48cbe9812a/sqlalchemy-2.1.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0534d3f41295a74d9fe2e0838e06ce6329a0855b2965df5f5d1c4d3f6f1406d0", size = 2468626, upload-time = "2026-10-21T18:49:33.602Z" }, + { url = "https://files.pythonhosted.org/packages/9b/98/1cccefb4fc94f4efd621604063acbbea4e3d530c27151fd7249d68adabf2/sqlalchemy-2.1.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:650f97996a4c529f4930969f3ba11bcdb57ff160ed1cb87ea0174393bf4f5f73", size = 4602545, upload-time = "2026-10-21T18:54:18.766Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/1817d7406ed92c8647a1d1db965a1bc2dd5a3f80d18e4d28ad62daf9bd72/sqlalchemy-2.1.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bada6bd29ae58a0581bf6977c0a43dfd67388fd09129f1475c37eca370cccd8", size = 4655491, upload-time = "2026-10-21T19:05:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/41/5c/0bdaa1676fb182b93e3c87c09cafd08a734ef29820aac6312b87a88f8537/sqlalchemy-2.1.6-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:96f5fcdfb7e89b6d895e2ddc165f67b3126a585dfdf2a95b0c1f3fcf3380d7d4", size = 4319667, upload-time = "2026-10-21T19:23:16.637Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d7/319e5a049b6f5235d1229858972abb05b8eae2fdd6639dc7bdbbc9cdb68d/sqlalchemy-2.1.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:015bd7b0687b5feaecb9988bc2f36ecb39daca83bd806b094b43b19ea503b0a9", size = 4525975, upload-time = "2026-10-21T18:54:21.553Z" }, + { url = "https://files.pythonhosted.org/packages/19/3b/6e8e5c14a3bcfae59a081a27f33e4f225df8140daab8e90fc3c42afd0d0e/sqlalchemy-2.1.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:bf883ae772971bc6155cb64cdfb9e31c50a2099f3b7c8b51b0401bf608a9fe8e", size = 4320900, upload-time = "2026-10-21T19:23:18.835Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a7/7ad220d5ea6308a9a32395785f4ce8beff4d00b0ae56ab9f2dafcca28743/sqlalchemy-2.1.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1ff897d13de613768123dee68474956a51fe641c42bba7af6cb7776716a8dbcb", size = 4616756, upload-time = "2026-10-21T19:05:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/08/81/53a198f5a29f247adc68c0c6da69bd2c7bdd89d3ffc58f9e626ee3160f18/sqlalchemy-2.1.6-cp313-cp313-win32.whl", hash = "sha256:9db8bacb2a69b4b5388bc1439463170af28f4cc710933085b81d1658d931d2c2", size = 2382042, upload-time = "2026-10-21T17:11:33.968Z" }, + { url = "https://files.pythonhosted.org/packages/b4/87/69e86a23cd9ccef0d654c09dc69dfea6d62b1311d2f13ca3daab1bcd922e/sqlalchemy-2.1.6-cp313-cp313-win_amd64.whl", hash = "sha256:b9f0b92b4a88b630f1e67df816b43cb017bb583479917dd100541d4f490a9517", size = 2432597, upload-time = "2026-10-21T17:11:35.514Z" }, + { url = "https://files.pythonhosted.org/packages/98/90/a7f81395135cc5da724403ece4340f7e04c1ce752da15d0ad337591ec9c9/sqlalchemy-2.1.6-cp313-cp313-win_arm64.whl", hash = "sha256:bc495d00f235c3df6f376187a4ef48a56e162fdcc0a113ee34518332aa1aa2df", size = 2393224, upload-time = "2026-10-21T18:54:18.405Z" }, + { url = "https://files.pythonhosted.org/packages/c1/69/c75534e336b06b3fd3f76de73d89736510800e60c77c58d4e4ce18389018/sqlalchemy-2.1.6-py3-none-any.whl", hash = "sha256:106878cf3795cd3d357e2e858609f9e44f20f19b2d8e48a3532bef3f90b03fbe", size = 2063055, upload-time = "2026-10-21T18:42:10.234Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 000000000..66aaf1189 --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,44 @@ +# The Simulation Entrypoint's only Alembic chain: the `simulation` schema. +# Run it from the persistence library's environment, for example: +# uv run --frozen --project libs/policyengine-simulation-persistence --extra migrations \ +# alembic -c migrations/alembic.ini upgrade head +# The database URL comes only from SIMULATION_MIGRATION_DATABASE_URL. +# Ownership rules: libs/policyengine-simulation-persistence/README.md + +[alembic] +script_location = %(here)s +path_separator = os +file_template = %%(rev)s_%%(slug)s + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 000000000..b024e1d89 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,113 @@ +"""Alembic environment for the Simulation Entrypoint's ``simulation`` schema.""" + +from __future__ import annotations + +import os +from logging.config import fileConfig +from typing import TYPE_CHECKING + +from alembic import context +from policyengine_simulation_persistence import ( + ALEMBIC_VERSION_TABLE, + SIMULATION_SCHEMA, + normalized_database_url, + simulation_metadata, +) +from sqlalchemy import create_engine, inspect, pool, text +from sqlalchemy.schema import CreateSchema + +if TYPE_CHECKING: + from sqlalchemy import Connection + +DATABASE_URL_VARIABLE = "SIMULATION_MIGRATION_DATABASE_URL" +CONNECT_TIMEOUT_SECONDS = 10 +# How long a migration statement waits for a lock before the migration fails. +# Adding a foreign key locks the Public API table it refers to, and the Public +# API's writes queue behind a migration that waits, so it must not wait long. +LOCK_TIMEOUT = "10s" + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name, disable_existing_loggers=False) + + +def _include_name(name: str | None, type_: str, _parent_names: object) -> bool: + """Compare only the simulation schema, never other services' schemas.""" + if type_ == "schema": + return name == SIMULATION_SCHEMA + return True + + +def _include_object( + _object: object, + name: str | None, + type_: str, + _reflected: object, + _compare_to: object, +) -> bool: + """Leave Alembic's own version table out of the schema comparison.""" + return not (type_ == "table" and name == ALEMBIC_VERSION_TABLE) + + +def _ensure_simulation_schema(connection: Connection) -> None: + """Create the simulation schema, owned by the migrator, if it is missing. + + PostgreSQL checks CREATE on the database before it checks whether a schema + exists, even for CREATE SCHEMA IF NOT EXISTS. Creating the schema only when + it is missing lets an operator create it in advance, owned by the migrator, + instead of granting the migrator CREATE on the database. + """ + if not inspect(connection).has_schema(SIMULATION_SCHEMA): + connection.execute(CreateSchema(SIMULATION_SCHEMA)) + + +def _run(connection: Connection) -> None: + connection.execute(text(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'")) + # The version table lives in the simulation schema, so the schema must + # exist before Alembic looks for it. + _ensure_simulation_schema(connection) + context.configure( + connection=connection, + target_metadata=simulation_metadata, + version_table=ALEMBIC_VERSION_TABLE, + version_table_schema=SIMULATION_SCHEMA, + include_schemas=True, + include_name=_include_name, + include_object=_include_object, + compare_type=True, + compare_server_default=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Apply migrations to the database named by the migration URL.""" + database_url = os.environ.get(DATABASE_URL_VARIABLE, "") + if not database_url: + msg = f"{DATABASE_URL_VARIABLE} must name the simulation database" + raise RuntimeError(msg) + engine = create_engine( + normalized_database_url(database_url), + poolclass=pool.NullPool, + connect_args={ + "connect_timeout": CONNECT_TIMEOUT_SECONDS, + "prepare_threshold": None, + }, + ) + try: + # One transaction covers the schema creation and every revision, so a + # failed migration leaves nothing half-applied. + with engine.begin() as connection: + _run(connection) + finally: + engine.dispose() + + +if context.is_offline_mode(): + msg = ( + "Simulation migrations run only online, against the database they " + "change; offline SQL generation is not supported" + ) + raise RuntimeError(msg) +run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 000000000..a56f67cc9 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +${imports if imports else ""} +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + """Apply this revision.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Revert this revision.""" + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/0001_initial_simulation_schema.py b/migrations/versions/0001_initial_simulation_schema.py new file mode 100644 index 000000000..038498dd3 --- /dev/null +++ b/migrations/versions/0001_initial_simulation_schema.py @@ -0,0 +1,52 @@ +"""Grant the Simulation Entrypoint runtime role access to the simulation schema. + +The environment creates the ``simulation`` schema, owned by the migrator, and +Alembic creates ``simulation.alembic_version`` before this revision runs. This +revision lets the runtime role use the schema, read the applied revision for +the readiness check, and read and write every table the migrator creates in +later revisions. The runtime role never owns or changes schema objects. + +Revision ID: 0001 +Revises: +Create Date: 2026-10-22 +""" + +from __future__ import annotations + +from alembic import op + +revision = "0001" +down_revision = None +branch_labels = None +depends_on = None + +# The statements name the schema and role literally: a migration is a fixed +# historical record, so later renames must not change what it did. +UPGRADE_STATEMENTS = ( + "GRANT USAGE ON SCHEMA simulation TO simulation_runtime", + "GRANT SELECT ON simulation.alembic_version TO simulation_runtime", + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "GRANT SELECT, INSERT, UPDATE ON TABLES TO simulation_runtime", + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "GRANT USAGE, SELECT ON SEQUENCES TO simulation_runtime", +) +DOWNGRADE_STATEMENTS = ( + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "REVOKE USAGE, SELECT ON SEQUENCES FROM simulation_runtime", + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "REVOKE SELECT, INSERT, UPDATE ON TABLES FROM simulation_runtime", + "REVOKE SELECT ON simulation.alembic_version FROM simulation_runtime", + "REVOKE USAGE ON SCHEMA simulation FROM simulation_runtime", +) + + +def upgrade() -> None: + """Grant schema usage, revision reads and default table privileges.""" + for statement in UPGRADE_STATEMENTS: + op.execute(statement) + + +def downgrade() -> None: + """Revoke everything ``upgrade`` granted.""" + for statement in DOWNGRADE_STATEMENTS: + op.execute(statement) diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 9783ef1cc..87ddd1f5e 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -22,6 +22,40 @@ Credential values belong in the cloud secret store. They must not be committed to the repository or stored as plain GitHub variables. The deployment workflow receives only the configuration needed to reference those protected resources. +## Simulation database + +The Simulation Entrypoint owns the `simulation` schema in the Postgres database +that holds the Public API's v2 tables. Before the entrypoint candidate deploys +in each environment, the `migrate_simulation_database` job applies the +schema's Alembic chain (`migrations/`) with the migrator role. It is the only +job that receives `SIMULATION_MIGRATION_DATABASE_URL`, a GitHub environment +secret in `beta` and `prod`, passed in that step's `env:` alone. If the +migration fails, the entrypoint is not deployed. Only a deployment from `main` +applies revisions; one dispatched from another branch (the Stage 12 deploy can +be) only requires the database to be at that branch's head, so an unmerged +revision never reaches a shared database. + +The deploy service account must be able to list the versions of the runtime +URL's secret (`gcloud secrets versions list`), as for the other runtime +secrets. + +The entrypoint itself connects as the runtime role. The deploy step resolves +the Secret Manager secret named by the `SIMULATION_DATABASE_URL_SECRET_NAME` +environment variable to its newest enabled version and mounts that version as +`SIMULATION_DATABASE_URL`. Only the entrypoint's runtime service account may +read it. Modal never receives either URL. + +`GET /ready` reports the database's applied revision next to the revision the +image expects, under `simulation_database`. The database does not change the +readiness status, because no old-gateway or Stage 12 route uses it. The +candidate smoke check requires the two revisions to match, which shows that the +runtime secret reached the revision and the migration ran. While a cold +candidate has not reached its database yet, it asks again, up to six requests +in all; the stable and +rollback checks do not, because an older image may run against a newer schema. +The ownership rules and roles are described in +`libs/policyengine-simulation-persistence/README.md`. + ## Candidate qualification For each eligible revision, the workflow first deploys `beta` diff --git a/projects/policyengine-simulation-entry/Dockerfile.dockerignore b/projects/policyengine-simulation-entry/Dockerfile.dockerignore index 9379db178..a54f9060f 100644 --- a/projects/policyengine-simulation-entry/Dockerfile.dockerignore +++ b/projects/policyengine-simulation-entry/Dockerfile.dockerignore @@ -13,6 +13,8 @@ libs/* !libs/policyengine-simulation-contract/** !libs/policyengine-simulation-observability/ !libs/policyengine-simulation-observability/** +!libs/policyengine-simulation-persistence/ +!libs/policyengine-simulation-persistence/** !libs/policyengine-stage12-persistence/ !libs/policyengine-stage12-persistence/** libs/**/tests diff --git a/projects/policyengine-simulation-entry/pyproject.toml b/projects/policyengine-simulation-entry/pyproject.toml index 1b43b4f1d..11f569523 100644 --- a/projects/policyengine-simulation-entry/pyproject.toml +++ b/projects/policyengine-simulation-entry/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "policyengine-observability[fastapi,google,httpx,otlp-grpc]>=3.0.4,<4", "policyengine-simulation-contract", "policyengine-simulation-observability", + "policyengine-simulation-persistence", "policyengine-stage12-persistence", "pyjwt>=2.10.1,<3", "uvicorn>=0.35.0,<1", @@ -30,6 +31,7 @@ packages = ["src/policyengine_simulation_entry"] [tool.uv.sources] policyengine-simulation-contract = { path = "../../libs/policyengine-simulation-contract", editable = true } policyengine-simulation-observability = { path = "../../libs/policyengine-simulation-observability", editable = true } +policyengine-simulation-persistence = { path = "../../libs/policyengine-simulation-persistence", editable = true } policyengine-stage12-persistence = { path = "../../libs/policyengine-stage12-persistence", editable = true } [project.optional-dependencies] diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index be3a699ce..d862ebcd1 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -25,7 +25,6 @@ JobSubmitResponse, PingRequest, PingResponse, - ReadinessResponse, SimulationErrorResponse, SimulationRequest, VersionMap, @@ -42,6 +41,7 @@ from policyengine_simulation_observability.observability import ( init_simulation_observability, ) +from policyengine_simulation_persistence import HEAD_REVISION from starlette.datastructures import MutableHeaders from policyengine_simulation_entry.auth import ( @@ -59,15 +59,19 @@ ComparisonBackend, EntryRuntime, ResponseIdentifier, + SimulationDatabaseConnection, model_json, request_identifiers, route_template, set_observability_header, ) from policyengine_simulation_entry.schemas import ( + SimulationDatabaseReadiness, + SimulationEntrypointReadinessResponse, TemporaryStage12ReportResponse, TemporaryStage12SubmissionResponse, ) +from policyengine_simulation_entry.simulation_database import SimulationDatabase from policyengine_simulation_entry.stage12_backend import ( Stage12ComparisonBackend, TemporaryStage12DispatchFailed, @@ -84,11 +88,13 @@ "STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS", "ComparisonBackend", "ResponseIdentifier", + "SimulationDatabaseConnection", "app", "create_app", ] logger = logging.getLogger(__name__) +SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS = 3.0 def _temporary_submission_payload( @@ -456,6 +462,34 @@ async def get_budget_window_job( ) +async def _simulation_database_readiness( + database: SimulationDatabaseConnection, +) -> SimulationDatabaseReadiness: + """Report the database's applied revision next to the image's.""" + try: + revision = await asyncio.wait_for( + database.read_revision(), + timeout=SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS, + ) + # Any failure to reach or read the database is reported, never raised: + # the old-gateway routes this check covers do not use it. + except Exception as error: # noqa: BLE001 # reported as unreachable + logger.warning( + "simulation_database_unreachable", + extra={"error_type": type(error).__name__}, + ) + return SimulationDatabaseReadiness( + reachable=False, + revision=None, + expected_revision=HEAD_REVISION, + ) + return SimulationDatabaseReadiness( + reachable=True, + revision=revision, + expected_revision=HEAD_REVISION, + ) + + def _register_service_routes(app: FastAPI, entry: EntryRuntime) -> None: """Add the public version, health, readiness and ping routes.""" @@ -488,14 +522,23 @@ async def get_country_versions(kind: str, request: Request) -> Response: async def health() -> HealthResponse: return HealthResponse() - @app.get("/ready", response_model=ReadinessResponse) + @app.get("/ready", response_model=SimulationEntrypointReadinessResponse) async def ready() -> Response: - if await entry.backend.ready(): - return JSONResponse(ReadinessResponse(status="ready").model_dump()) - return JSONResponse( - ReadinessResponse(status="not_ready").model_dump(), - status_code=503, + backend_ready = await entry.backend.ready() + status = "ready" if backend_ready else "not_ready" + status_code = 200 if backend_ready else 503 + if entry.database is None: + # Without a configured database the response keeps its old shape. + readiness = SimulationEntrypointReadinessResponse(status=status) + return JSONResponse( + readiness.model_dump(mode="json", exclude={"simulation_database"}), + status_code=status_code, + ) + readiness = SimulationEntrypointReadinessResponse( + status=status, + simulation_database=await _simulation_database_readiness(entry.database), ) + return JSONResponse(readiness.model_dump(mode="json"), status_code=status_code) @app.post( "/ping", @@ -515,9 +558,15 @@ def create_app( backend: SimulationBackend | None = None, auth_dependency: AuthenticationDependency | None = None, comparison_backend: ComparisonBackend | None = None, + simulation_database: SimulationDatabaseConnection | None = None, ) -> FastAPI: """Build the app with injectable auth/backend seams for hermetic tests.""" runtime_settings = settings or Settings.from_env() + runtime_database = simulation_database + if runtime_database is None and runtime_settings.simulation_database_url: + runtime_database = SimulationDatabase.from_url( + runtime_settings.simulation_database_url + ) @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: @@ -527,6 +576,8 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: yield finally: await runtime_backend.close() + if runtime_database is not None: + runtime_database.dispose() runtime.shutdown() app = FastAPI( @@ -565,6 +616,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: runtime=runtime, backend=runtime_backend, comparison=runtime_comparison, + database=runtime_database, ) _install_request_context(app, runtime_settings) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py index 8385097e9..5d23e8c54 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py @@ -92,6 +92,10 @@ class Settings: stage12_database_url: str = "" stage12_artifact_bucket: str = "" stage12_retention_days: int = REQUIRED_STAGE12_RETENTION_DAYS + # The runtime role's URL for the Simulation Entrypoint's own schema. Empty + # outside deployed environments; nothing on the old-gateway or Stage 12 + # paths reads it. + simulation_database_url: str = "" @classmethod def from_env(cls) -> Settings: @@ -142,6 +146,7 @@ def from_env(cls) -> Settings: stage12_database_url=os.getenv("STAGE12_DATABASE_URL", ""), stage12_artifact_bucket=os.getenv("STAGE12_ARTIFACT_BUCKET", ""), stage12_retention_days=int(os.getenv("STAGE12_RETENTION_DAYS", "30")), + simulation_database_url=os.getenv("SIMULATION_DATABASE_URL", ""), ) @property @@ -170,6 +175,7 @@ def validate(self) -> None: self._validate_caller_auth() self._validate_old_gateway() self._validate_stage12() + self._validate_simulation_database() def _validate_caller_auth(self) -> None: """Require caller authentication in production, with an HTTPS issuer.""" @@ -267,3 +273,11 @@ def _validate_stage12(self) -> None: ): msg = "STAGE12_DATABASE_URL must use the canonical postgresql:// scheme." raise ConfigurationError(msg) + + def _validate_simulation_database(self) -> None: + """Require the canonical scheme for the schema's runtime URL, when set.""" + if self.simulation_database_url and not self.simulation_database_url.startswith( + "postgresql://" + ): + msg = "SIMULATION_DATABASE_URL must use the canonical postgresql:// scheme." + raise ConfigurationError(msg) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py index 279700705..4898f758f 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py @@ -210,6 +210,21 @@ def _add_response_telemetry( attributes[response_identifier_key] = response_identifier +class SimulationDatabaseConnection(Protocol): + """The Simulation Entrypoint's own database, as the routes use it. + + The real implementation is ``SimulationDatabase``; tests inject fakes. + """ + + async def read_revision(self) -> str | None: + """Return the applied migration revision, or None before the first one.""" + ... + + def dispose(self) -> None: + """Close every pooled connection.""" + ... + + class EntryRuntime: """The settings, telemetry runtime and backends the entry routes share.""" @@ -220,12 +235,14 @@ def __init__( runtime: ObservabilityRuntime, backend: SimulationBackend, comparison: ComparisonBackend | None, + database: SimulationDatabaseConnection | None, ) -> None: """Keep the dependencies that ``create_app`` resolved.""" self.settings = settings self.runtime = runtime self.backend = backend self.comparison = comparison + self.database = database def bind_observability_id(self, request: Request, value: object) -> str | None: """Bind a normalized observability ID to the request and telemetry context.""" diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py index d387526df..74aa0f74e 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py @@ -115,3 +115,28 @@ class TemporaryStage12ReportResponse(BaseModel): report: ComparisonReportRecord simulations: tuple[ComparisonSimulationRecord, ...] + + +class SimulationDatabaseReadiness(BaseModel): + """The Simulation Entrypoint's own database, as the readiness check found it. + + ``revision`` is the migration revision the database has applied, or null + when it was unreachable or has no migrations yet; ``expected_revision`` is + the newest revision this image knows. Neither changes ``status``: no + old-gateway or Stage 12 route uses this database. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + reachable: bool + revision: str | None + expected_revision: str + + +class SimulationEntrypointReadinessResponse(BaseModel): + """Old-gateway readiness, plus the database revision when one is configured.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + status: Literal["ready", "not_ready"] + simulation_database: SimulationDatabaseReadiness | None = None diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py new file mode 100644 index 000000000..e0ca4f1bd --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py @@ -0,0 +1,42 @@ +"""The Simulation Entrypoint's connection to its own ``simulation`` schema.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from policyengine_simulation_persistence import ( + create_simulation_engine, + read_schema_revision, +) + +if TYPE_CHECKING: + from sqlalchemy import Engine + + +class SimulationDatabase: + """Run short, blocking database operations off the event loop.""" + + def __init__(self, engine: Engine) -> None: + """Use ``engine``, which connects as the runtime role.""" + self.engine = engine + + @classmethod + def from_url(cls, database_url: str) -> SimulationDatabase: + """Build the runtime engine; no connection opens until first use. + + Raise ValueError, without echoing the URL, when it is malformed. + """ + return cls(create_simulation_engine(database_url)) + + def _read_revision(self) -> str | None: + with self.engine.connect() as connection: + return read_schema_revision(connection) + + async def read_revision(self) -> str | None: + """Return the applied migration revision, or None before the first one.""" + return await asyncio.to_thread(self._read_revision) + + def dispose(self) -> None: + """Close every pooled connection.""" + self.engine.dispose() diff --git a/projects/policyengine-simulation-entry/tests/deployment_rule_checks.py b/projects/policyengine-simulation-entry/tests/deployment_rule_checks.py new file mode 100644 index 000000000..fa06bed25 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/deployment_rule_checks.py @@ -0,0 +1,55 @@ +"""Checks behind two deployment rules, runnable on the repository or a fixture. + +- ``combined_secret_resolutions`` (sim.deploy.workflow-shell-safety): Bash + gives an array assignment the exit status of its last command substitution + only, so an assignment that resolves several secrets lets an earlier + failure pass unnoticed. +- ``libraries_missing_from_context`` (sim.deploy.container-context-minimal): + every local library the entrypoint installs must be in its build context. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import tomllib + +ARRAY_ASSIGNMENT = re.compile(r"^\s*\w+\+?=\(", re.MULTILINE) +# The line that closes a multi-line array assignment: only its parenthesis. +ARRAY_END = re.compile(r"\n[ \t]*\)") +SECRET_RESOLUTION = "$(resolve_secret_reference" + + +def combined_secret_resolutions(script_text: str) -> list[str]: + """Return array assignments that resolve more than one secret.""" + violations = [] + for match in ARRAY_ASSIGNMENT.finditer(script_text): + line_end = script_text.find("\n", match.end()) + first_line = script_text[match.start() : line_end if line_end >= 0 else None] + if ")" in first_line: + assignment = first_line + else: + closing = ARRAY_END.search(script_text, match.end()) + assignment = script_text[ + match.start() : closing.start() if closing is not None else None + ] + if assignment.count(SECRET_RESOLUTION) > 1: + violations.append(assignment.strip().splitlines()[0]) + return violations + + +def libraries_missing_from_context(pyproject_text: str, ignore_text: str) -> list[str]: + """Return the local libraries the project installs but its context leaves out.""" + rules = ignore_text.splitlines() + sources = ( + tomllib.loads(pyproject_text).get("tool", {}).get("uv", {}).get("sources", {}) + ) + libraries = [ + Path(source["path"]).name for source in sources.values() if "path" in source + ] + return [ + library + for library in libraries + if f"!libs/{library}/" not in rules or f"!libs/{library}/**" not in rules + ] diff --git a/projects/policyengine-simulation-entry/tests/migration_wiring_checks.py b/projects/policyengine-simulation-entry/tests/migration_wiring_checks.py new file mode 100644 index 000000000..8b899767d --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/migration_wiring_checks.py @@ -0,0 +1,67 @@ +"""The check behind sim.persistence.migrate-before-entrypoint's workflow part. + +It takes a deploy workflow's jobs, so the rule test runs it on the reusable +deploy workflow and the proof test on the rule's fixtures. +""" + +from __future__ import annotations + +import json + +MIGRATION_JOB = "migrate_simulation_database" +MIGRATION_STEP = "Apply simulation schema migrations" +MIGRATION_SECRET = "SIMULATION_MIGRATION_DATABASE_URL" +ENTRYPOINT_JOB = "deploy_entrypoint" +MIGRATION_MODE = "${{ github.ref == 'refs/heads/main' && 'upgrade' || 'verify' }}" + + +def _needs(job: dict) -> list[str]: + needs = job.get("needs", []) + return [needs] if isinstance(needs, str) else list(needs) + + +def secret_carriers( + jobs: dict, *, workflow_env: dict | None = None +) -> list[tuple[str, str]]: + """Return each job, job env or step that holds the migrator URL. + + A workflow-level ``env:`` reaches every job, so it counts as a carrier + for all of them. + """ + carriers = [] + if MIGRATION_SECRET in json.dumps(workflow_env or {}): + carriers.append(("workflow", "workflow env")) + for job_name, job in jobs.items(): + if MIGRATION_SECRET in json.dumps(job.get("env", {})): + carriers.append((job_name, "job env")) + carriers.extend( + (job_name, str(step.get("name"))) + for step in job.get("steps", []) + if MIGRATION_SECRET in json.dumps(step) + ) + return carriers + + +def migration_wiring_violations( + jobs: dict, *, workflow_env: dict | None = None +) -> list[str]: + """Return how a deploy workflow's jobs depart from the migration wiring.""" + migration = jobs.get(MIGRATION_JOB) + if migration is None: + return [f"no {MIGRATION_JOB} job"] + violations = [] + entrypoint = jobs.get(ENTRYPOINT_JOB, {}) + if MIGRATION_JOB not in _needs(entrypoint) or ( + f"needs.{MIGRATION_JOB}.result == 'success'" not in entrypoint.get("if", "") + ): + violations.append( + f"{ENTRYPOINT_JOB} does not wait for a successful {MIGRATION_JOB}" + ) + carriers = secret_carriers(jobs, workflow_env=workflow_env) + if carriers != [(MIGRATION_JOB, MIGRATION_STEP)]: + violations.append(f"{MIGRATION_SECRET} reaches {carriers}") + steps = {step.get("name"): step for step in migration.get("steps", [])} + mode = steps.get(MIGRATION_STEP, {}).get("env", {}).get("SIMULATION_MIGRATION_MODE") + if mode != MIGRATION_MODE: + violations.append(f"{MIGRATION_STEP} applies revisions from any branch") + return violations diff --git a/projects/policyengine-simulation-entry/tests/test_config.py b/projects/policyengine-simulation-entry/tests/test_config.py index 8dbf6809f..23ed45320 100644 --- a/projects/policyengine-simulation-entry/tests/test_config.py +++ b/projects/policyengine-simulation-entry/tests/test_config.py @@ -297,3 +297,41 @@ def test_validation_reports_the_first_failed_check(overrides, message): ) def test_consistent_configuration_validates(overrides): make_settings(**overrides).validate() + + +def test_the_simulation_database_is_optional(monkeypatch): + from policyengine_simulation_entry.config import Settings + + monkeypatch.delenv("SIMULATION_DATABASE_URL", raising=False) + settings = Settings.from_env() + + assert settings.simulation_database_url == "" + make_settings(simulation_database_url="").validate() + + +def test_the_simulation_database_url_is_read_from_the_environment(monkeypatch): + from policyengine_simulation_entry.config import Settings + + monkeypatch.setenv( + "SIMULATION_DATABASE_URL", + "postgresql://simulation_runtime:secret@db.example:5432/postgres", + ) + + assert Settings.from_env().simulation_database_url == ( + "postgresql://simulation_runtime:secret@db.example:5432/postgres" + ) + + +@pytest.mark.parametrize( + "database_url", + [ + "postgres://simulation_runtime:secret@db.example/postgres", + "postgresql+psycopg://simulation_runtime:secret@db.example/postgres", + "mysql://simulation_runtime:secret@db.example/postgres", + ], +) +def test_the_simulation_database_url_must_be_canonical(database_url): + with pytest.raises(ConfigurationError, match="SIMULATION_DATABASE_URL") as error: + make_settings(simulation_database_url=database_url).validate() + + assert "secret" not in str(error.value) diff --git a/projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py b/projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py index 81d7ad590..ad60f091d 100644 --- a/projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py +++ b/projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py @@ -23,8 +23,9 @@ / "sim.boundaries.sqlalchemy-and-alembic-only" ) # Every directory that holds non-test source. The executor's src/modal is -# under projects/*/src. -SOURCE_ROOT_PATTERNS = ("projects/*/src", "libs/*/src", ".github/scripts") +# under projects/*/src. The simulation schema's Alembic chain may import +# alembic and change the schema, but still connects only through SQLAlchemy. +SOURCE_ROOT_PATTERNS = ("projects/*/src", "libs/*/src", ".github/scripts", "migrations") # Directories the rule names, each of which must contribute scanned files. REQUIRED_SOURCE_DIRECTORIES = ( "projects/policyengine-simulation-entry/src", @@ -34,8 +35,10 @@ "libs/policyengine-fastapi/src", "libs/policyengine-simulation-contract/src", "libs/policyengine-simulation-observability/src", + "libs/policyengine-simulation-persistence/src", "libs/policyengine-stage12-persistence/src", ".github/scripts", + "migrations", ) diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index a47600cbc..804fae52b 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -210,8 +210,9 @@ def _assert_beta_then_prod_release( def _assert_deployment_environment_settings(reusable_workflow: str) -> None: assert ( - # Sentinel, restore and final routing acknowledgement use secrets too. - reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 15 + # Sentinel, restore, final routing acknowledgement and the simulation + # database migration use secrets too. + reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 16 ) assert "APP_ENVIRONMENT: ${{ inputs.deployment_environment }}" in reusable_workflow assert "STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}" in reusable_workflow diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py index 1052bfab5..a6d870841 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py @@ -10,10 +10,12 @@ from pathlib import Path import pytest +from deployment_rule_checks import SECRET_RESOLUTION, combined_secret_resolutions from test_deployment_assets_support import ( CLOUD_RUN_DEPLOY_SCRIPT, DEPLOY_WORKFLOW, DOCKERFILE, + REPOSITORY_ROOT, REUSABLE_DEPLOY_WORKFLOW, STAGE12_DEPLOY_WORKFLOW, ) @@ -39,7 +41,7 @@ def test_cloud_run_deployment_does_not_interpolate_github_values_in_shell(): assert ":latest" not in deployment_script -def _cloud_run_deploy_environment(tmp_path: Path) -> dict[str, str]: +def cloud_run_deploy_environment(tmp_path: Path) -> dict[str, str]: """Return a complete deployment environment backed by a fake gcloud.""" fake_bin = tmp_path / "bin" fake_bin.mkdir() @@ -114,11 +116,14 @@ def _cloud_run_deploy_environment(tmp_path: Path) -> dict[str, str]: "STAGE12_DATABASE_URL_SECRET_NAME": "stage12-staging-database-url", "MODAL_TOKEN_ID_SECRET_NAME": "modal-token-id", "MODAL_TOKEN_SECRET_SECRET_NAME": "modal-token-secret", + "SIMULATION_DATABASE_URL_SECRET_NAME": ( + "simulation-staging-runtime-database-url" + ), } def test_cloud_run_deployment_escapes_environment_and_pins_secret_versions(tmp_path): - command_environment = _cloud_run_deploy_environment(tmp_path) + command_environment = cloud_run_deploy_environment(tmp_path) arguments_capture = Path(command_environment["GCLOUD_ARGUMENTS_CAPTURE"]) env_capture = Path(command_environment["GCLOUD_ENV_FILE_CAPTURE"]) @@ -152,6 +157,7 @@ def test_cloud_run_deployment_escapes_environment_and_pins_secret_versions(tmp_p secrets_argument = arguments[arguments.index("--set-secrets") + 1] assert secrets_argument == ( "OLD_GATEWAY_AUTH_CLIENT_SECRET=old-client-secret:3," + "SIMULATION_DATABASE_URL=simulation-staging-runtime-database-url:3," "MODAL_TOKEN_ID=modal-token-id:3," "MODAL_TOKEN_SECRET=modal-token-secret:3," "STAGE12_DATABASE_URL=stage12-staging-database-url:3" @@ -159,6 +165,75 @@ def test_cloud_run_deployment_escapes_environment_and_pins_secret_versions(tmp_p assert ":latest" not in secrets_argument +@pytest.mark.parametrize( + "secret_name_variable", + [ + "OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME", + "SIMULATION_DATABASE_URL_SECRET_NAME", + "MODAL_TOKEN_ID_SECRET_NAME", + "MODAL_TOKEN_SECRET_SECRET_NAME", + "STAGE12_DATABASE_URL_SECRET_NAME", + ], +) +def test_cloud_run_deployment_stops_when_a_runtime_secret_cannot_be_resolved( + tmp_path, secret_name_variable +): + command_environment = cloud_run_deploy_environment(tmp_path) | { + secret_name_variable: "not a secret name" + } + + result = subprocess.run( + [CLOUD_RUN_DEPLOY_SCRIPT], + check=False, + capture_output=True, + text=True, + env=command_environment, + ) + + assert result.returncode != 0 + assert "invalid Secret Manager reference" in result.stderr + assert not Path(command_environment["GCLOUD_ARGUMENTS_CAPTURE"]).exists() + + +def test_cloud_run_deployment_resolves_each_secret_in_its_own_assignment(): + script = Path(CLOUD_RUN_DEPLOY_SCRIPT).read_text(encoding="utf-8") + + assert SECRET_RESOLUTION in script + assert combined_secret_resolutions(script) == [] + + +def test_rule_proof_workflow_shell_safety(): + """Proof for rule sim.deploy.workflow-shell-safety (rules/registry.toml).""" + fixtures = ( + REPOSITORY_ROOT / "rules" / "fixtures" / ("sim.deploy.workflow-shell-safety") + ) + + assert combined_secret_resolutions( + (fixtures / "bad_deploy_script.sh").read_text(encoding="utf-8") + ) == ["runtime_secrets=("] + assert ( + combined_secret_resolutions( + (fixtures / "good_deploy_script.sh").read_text(encoding="utf-8") + ) + == [] + ) + + +def test_stage12_secrets_are_resolved_by_the_stage12_block_alone(): + """The comparison path shares no deploy helper with the main path.""" + script = Path(CLOUD_RUN_DEPLOY_SCRIPT).read_text(encoding="utf-8") + stage12_block = script.split('if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then', 1)[ + 1 + ].split("\nfi\n", 1)[0] + main_path = script.replace(stage12_block, "") + + assert "add_stage12_runtime_secret STAGE12_DATABASE_URL" in stage12_block + assert "add_runtime_secret " not in stage12_block + assert "add_stage12_runtime_secret " not in main_path.replace( + "add_stage12_runtime_secret() {", "" + ) + + @pytest.mark.parametrize( ("configured", "expected"), [(None, "0"), ("", "0"), ("0", "0"), ("1", "1")], @@ -169,7 +244,7 @@ def test_cloud_run_deployment_defaults_permission_enforcement_to_log_only( configured, expected, ): - command_environment = _cloud_run_deploy_environment(tmp_path) + command_environment = cloud_run_deploy_environment(tmp_path) if configured is not None: command_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE"] = ( configured @@ -192,7 +267,7 @@ def test_cloud_run_deployment_rejects_invalid_permission_enforcement( tmp_path, configured, ): - command_environment = _cloud_run_deploy_environment(tmp_path) + command_environment = cloud_run_deploy_environment(tmp_path) command_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE"] = configured result = subprocess.run( diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py index ae004c799..00ab839cf 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py @@ -128,7 +128,22 @@ def write_fake_smoke_curl(tmp_path: Path, revisions: list[str]) -> tuple[Path, P printf '{"status":"healthy"}' > "${body_file}" ;; */ready) - printf '{"status":"ready"}' + ready_body='{"status":"ready"}' + if [ -n "${FAKE_CURL_READY_BODIES:-}" ]; then + # One body per line, served in order; the last one repeats. + ready_count=0 + if [ -f "${FAKE_CURL_READY_BODIES}.count" ]; then + ready_count="$(cat "${FAKE_CURL_READY_BODIES}.count")" + fi + ready_count=$((ready_count + 1)) + printf '%s' "${ready_count}" > "${FAKE_CURL_READY_BODIES}.count" + line_count="$(wc -l < "${FAKE_CURL_READY_BODIES}")" + if [ "${ready_count}" -gt "${line_count}" ]; then + ready_count="${line_count}" + fi + ready_body="$(sed -n "${ready_count}p" "${FAKE_CURL_READY_BODIES}")" + fi + printf '%s' "${FAKE_CURL_READY:-${ready_body}}" ;; */versions) versions='{}' diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py index cda498290..df2e4e669 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py @@ -6,6 +6,7 @@ import textwrap import tomllib +from deployment_rule_checks import libraries_missing_from_context from test_deployment_assets_support import ( DOCKERIGNORE, GITIGNORE, @@ -172,6 +173,40 @@ def test_container_context_excludes_local_environments_and_unrelated_projects(): ) +def test_container_context_includes_every_local_library_the_entrypoint_installs(): + entry_project = REPOSITORY_ROOT / "projects" / "policyengine-simulation-entry" + pyproject = (entry_project / "pyproject.toml").read_text(encoding="utf-8") + + assert "path = " in pyproject + assert ( + libraries_missing_from_context( + pyproject, DOCKERIGNORE.read_text(encoding="utf-8") + ) + == [] + ) + + +def test_rule_proof_container_context_minimal(): + """Proof for rule sim.deploy.container-context-minimal (rules/registry.toml).""" + fixtures = ( + REPOSITORY_ROOT + / "rules" + / "fixtures" + / ("sim.deploy.container-context-minimal") + ) + pyproject = (fixtures / "pyproject.toml").read_text(encoding="utf-8") + + assert libraries_missing_from_context( + pyproject, (fixtures / "bad.dockerignore").read_text(encoding="utf-8") + ) == ["fixture-persistence"] + assert ( + libraries_missing_from_context( + pyproject, (fixtures / "good.dockerignore").read_text(encoding="utf-8") + ) + == [] + ) + + def test_production_lock_contains_the_direct_stage12_database_runtime(): lockfile = tomllib.loads( ( diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index 060b6bb8c..65e79cf45 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -122,6 +122,22 @@ def test_local_area_breakdowns_are_public(): ]["properties"]["result"]["anyOf"] +def test_readiness_documents_the_simulation_database_revision(): + spec = create_app().openapi() + readiness = spec["paths"]["/ready"]["get"]["responses"]["200"] + schema = readiness["content"]["application/json"]["schema"] + database = spec["components"]["schemas"]["SimulationDatabaseReadiness"] + + assert schema == { + "$ref": "#/components/schemas/SimulationEntrypointReadinessResponse" + } + assert set(database["properties"]) == { + "reachable", + "revision", + "expected_revision", + } + + def test_route_table_is_frozen(): assert methods(cast("OpenAPIDocument", create_app().openapi())) == EXPECTED_ROUTES @@ -161,7 +177,9 @@ def test_normalized_contract_matches_old_gateway(): cloud_run_spec ) == normalized_compatibility_paths(gateway_spec) cloud_run_schemas = deepcopy(cloud_run_spec["components"]["schemas"]) - cloud_run_schemas.pop("ReadinessResponse") + # The entrypoint's readiness response adds its own database's revision. + cloud_run_schemas.pop("SimulationEntrypointReadinessResponse") + cloud_run_schemas.pop("SimulationDatabaseReadiness") cloud_run_schemas.pop("SimulationErrorResponse") assert cloud_run_schemas == gateway_spec["components"]["schemas"] gateway_operation_ids = operation_ids(gateway_spec) diff --git a/projects/policyengine-simulation-entry/tests/test_simulation_database.py b/projects/policyengine-simulation-entry/tests/test_simulation_database.py new file mode 100644 index 000000000..95b1defae --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_simulation_database.py @@ -0,0 +1,184 @@ +"""The Simulation Entrypoint's own database: readiness reporting and lifetime.""" + +from __future__ import annotations + +import asyncio + +import pytest +from conftest import FakeBackend, make_settings +from fastapi.testclient import TestClient +from policyengine_simulation_entry import app as app_module +from policyengine_simulation_entry import simulation_database as database_module +from policyengine_simulation_entry.app import create_app +from policyengine_simulation_entry.simulation_database import SimulationDatabase +from policyengine_simulation_persistence import HEAD_REVISION +from sqlalchemy.exc import OperationalError + +DATABASE_URL = "postgresql://simulation_runtime:secret@db.example:5432/postgres" + + +class FakeSimulationDatabase: + """Report a fixed revision, or fail the way an unreachable database does.""" + + def __init__(self, *, revision=HEAD_REVISION, failure=None, delay_seconds=0): + self.revision = revision + self.failure = failure + self.delay_seconds = delay_seconds + self.disposed = False + + async def read_revision(self): + if self.delay_seconds: + await asyncio.sleep(self.delay_seconds) + if self.failure is not None: + raise self.failure + return self.revision + + def dispose(self): + self.disposed = True + + +def _client(backend, database): + return TestClient( + create_app( + settings=make_settings(simulation_database_url=DATABASE_URL), + backend=backend, + auth_dependency=lambda: None, + simulation_database=database, + ) + ) + + +def test_readiness_reports_the_database_revision_and_the_expected_one(): + backend = FakeBackend() + with _client(backend, FakeSimulationDatabase(revision="0001")) as client: + result = client.get("/ready") + + assert result.status_code == 200 + assert result.json() == { + "status": "ready", + "simulation_database": { + "reachable": True, + "revision": "0001", + "expected_revision": HEAD_REVISION, + }, + } + + +def test_readiness_reports_a_database_that_has_no_migrations_yet(): + with _client(FakeBackend(), FakeSimulationDatabase(revision=None)) as client: + database = client.get("/ready").json()["simulation_database"] + + assert database == { + "reachable": True, + "revision": None, + "expected_revision": HEAD_REVISION, + } + + +def test_an_unreachable_database_does_not_change_old_gateway_readiness(caplog): + failure = OperationalError("SELECT 1", {}, Exception("password=secret")) + with _client(FakeBackend(), FakeSimulationDatabase(failure=failure)) as client: + result = client.get("/ready") + + assert result.status_code == 200 + assert result.json() == { + "status": "ready", + "simulation_database": { + "reachable": False, + "revision": None, + "expected_revision": HEAD_REVISION, + }, + } + assert "secret" not in caplog.text + + +def test_readiness_waits_only_briefly_for_the_database(monkeypatch): + monkeypatch.setattr( + app_module, "SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS", 0.01 + ) + slow_database = FakeSimulationDatabase(delay_seconds=1) + with _client(FakeBackend(), slow_database) as client: + database = client.get("/ready").json()["simulation_database"] + + assert database["reachable"] is False + + +def test_readiness_still_follows_the_old_gateway(): + backend = FakeBackend() + backend.is_ready = False + with _client(backend, FakeSimulationDatabase()) as client: + result = client.get("/ready") + + assert result.status_code == 503 + assert result.json()["status"] == "not_ready" + assert result.json()["simulation_database"]["reachable"] is True + + +def test_the_database_is_released_at_shutdown(): + database = FakeSimulationDatabase() + with _client(FakeBackend(), database): + assert database.disposed is False + + assert database.disposed is True + + +def test_settings_build_the_database_only_when_it_is_configured(monkeypatch): + built_urls = [] + + def build(database_url): + built_urls.append(database_url) + return FakeSimulationDatabase() + + monkeypatch.setattr(SimulationDatabase, "from_url", staticmethod(build)) + + create_app(settings=make_settings(), backend=FakeBackend()) + assert built_urls == [] + + create_app( + settings=make_settings(simulation_database_url=DATABASE_URL), + backend=FakeBackend(), + ) + assert built_urls == [DATABASE_URL] + + +def test_the_database_reads_the_revision_without_blocking_the_event_loop( + monkeypatch, +): + connections = [] + + class FakeConnection: + def __enter__(self): + connections.append(self) + return self + + def __exit__(self, *_): + return False + + class FakeEngine: + disposed = False + + def connect(self): + return FakeConnection() + + def dispose(self): + self.disposed = True + + monkeypatch.setattr( + database_module, + "read_schema_revision", + lambda connection: "0001" if connection in connections else None, + ) + engine = FakeEngine() + database = SimulationDatabase(engine) + + assert asyncio.run(database.read_revision()) == "0001" + database.dispose() + assert engine.disposed is True + + +@pytest.mark.parametrize("database_url", ["", "postgres://user:secret@db/postgres"]) +def test_a_malformed_database_url_never_builds_an_engine(database_url): + with pytest.raises(ValueError, match="Simulation database URL") as error: + SimulationDatabase.from_url(database_url) + + assert "secret" not in str(error.value) diff --git a/projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py b/projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py new file mode 100644 index 000000000..ee6486c0e --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py @@ -0,0 +1,442 @@ +"""Deployment wiring for the Simulation Entrypoint's own database. + +The migration job applies the chain before the entrypoint candidate deploys, +with the migrator URL in that job alone; the entrypoint receives the runtime +URL as a pinned Cloud Run secret; and the candidate smoke requires the +database to be at the image's revision. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest +import yaml +from migration_wiring_checks import migration_wiring_violations, secret_carriers +from test_deployment_assets_cloud_run import cloud_run_deploy_environment +from test_deployment_assets_support import ( + CLOUD_RUN_DEPLOY_SCRIPT, + REUSABLE_DEPLOY_WORKFLOW, + SMOKE_SCRIPT, + evaluate_github_expression, + smoke_test_env, + workflow_jobs, + write_fake_smoke_curl, +) +from workflow_script_support import ( + REPOSITORY_ROOT, + WORKFLOWS_DIR, + FakeCommands, + run_script, +) + +MIGRATION_SCRIPT = "migrate-simulation-database.sh" +MIGRATION_JOB = "migrate_simulation_database" +MIGRATION_SECRET = "SIMULATION_MIGRATION_DATABASE_URL" +FAKE_MIGRATION_URL = "postgresql://simulation_migrator:secret@db.example/postgres" + + +@pytest.fixture +def fakes(tmp_path: Path) -> FakeCommands: + return FakeCommands.create(tmp_path) + + +# --- The migration job ---------------------------------------------------- + + +def test_the_migration_job_runs_before_the_entrypoint_deploys(): + jobs = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + migration = jobs[MIGRATION_JOB] + deploy_entrypoint = jobs["deploy_entrypoint"] + + assert migration["needs"] == "prepare" + assert migration["environment"] == "${{ inputs.release_environment }}" + assert MIGRATION_JOB in deploy_entrypoint["needs"] + context = { + "inputs": {"deploy_stage12_v2": True}, + "needs": { + "prepare": {"result": "success"}, + "configure_stage12_v2": {"result": "success"}, + MIGRATION_JOB: {"result": "success"}, + }, + } + assert evaluate_github_expression(deploy_entrypoint["if"], context, job_if=True) + for failed_result in ("failure", "skipped", "cancelled"): + context["needs"][MIGRATION_JOB]["result"] = failed_result + assert not evaluate_github_expression( + deploy_entrypoint["if"], context, job_if=True + ) + + +def test_the_migration_job_applies_the_chain_with_the_migrator_url_alone(): + migration = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)[MIGRATION_JOB] + steps = {step.get("name"): step for step in migration["steps"]} + install = steps["Install migration dependencies"] + apply_step = steps["Apply simulation schema migrations"] + + assert install["working-directory"] == "libs/policyengine-simulation-persistence" + assert install["run"] == "uv sync --frozen --extra migrations" + assert apply_step["working-directory"] == "libs/policyengine-simulation-persistence" + assert apply_step["run"] == f"../../.github/scripts/{MIGRATION_SCRIPT}" + assert apply_step["env"] == { + MIGRATION_SECRET: "${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }}", + "SIMULATION_MIGRATION_MODE": ( + "${{ github.ref == 'refs/heads/main' && 'upgrade' || 'verify' }}" + ), + } + assert "env" not in migration + assert "permissions" not in migration + + +def _workflow(path: Path) -> dict: + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def test_only_the_migration_step_receives_the_migrator_url(): + carriers = [ + (path.name, *carrier) + for path in sorted(WORKFLOWS_DIR.glob("*.yml")) + for carrier in secret_carriers( + _workflow(path)["jobs"], workflow_env=_workflow(path).get("env") + ) + ] + + assert carriers == [ + ( + "simulation-deploy.reusable.yml", + MIGRATION_JOB, + "Apply simulation schema migrations", + ) + ] + + +def test_the_deploy_workflow_follows_the_migration_wiring(): + workflow = _workflow(REUSABLE_DEPLOY_WORKFLOW) + + assert ( + migration_wiring_violations(workflow["jobs"], workflow_env=workflow.get("env")) + == [] + ) + + +def test_the_migrator_url_may_not_reach_every_job_through_workflow_env(): + workflow = yaml.safe_load( + ( + REPOSITORY_ROOT + / "rules" + / "fixtures" + / "sim.persistence.migrate-before-entrypoint" + / "good_workflow.yml" + ).read_text(encoding="utf-8") + ) + workflow["env"] = { + MIGRATION_SECRET: "${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }}" + } + + assert migration_wiring_violations(workflow["jobs"], workflow_env=workflow["env"]) + + +def test_rule_proof_migrate_before_entrypoint(): + """Proof for rule sim.persistence.migrate-before-entrypoint.""" + fixtures = ( + REPOSITORY_ROOT + / "rules" + / "fixtures" + / ("sim.persistence.migrate-before-entrypoint") + ) + + bad = migration_wiring_violations( + yaml.safe_load((fixtures / "bad_workflow.yml").read_text(encoding="utf-8"))[ + "jobs" + ] + ) + good = migration_wiring_violations( + yaml.safe_load((fixtures / "good_workflow.yml").read_text(encoding="utf-8"))[ + "jobs" + ] + ) + + assert len(bad) == 3 + assert good == [] + + +UV_ALEMBIC = ["uv", "run", "--frozen", "--extra", "migrations", "alembic", "-c", + "../../migrations/alembic.ini"] # fmt: skip + + +def _run_migration_script(fakes, **environment): + values = { + "SIMULATION_MIGRATION_DATABASE_URL": FAKE_MIGRATION_URL, + "SIMULATION_MIGRATION_MODE": "upgrade", + **environment, + } + return run_script(MIGRATION_SCRIPT, env=fakes.env(**values)) + + +def test_the_migration_script_upgrades_and_then_requires_the_head(fakes): + fakes.add("uv") + + result = _run_migration_script(fakes) + + assert result.returncode == 0, result.stderr + assert fakes.calls("uv") == [ + [*UV_ALEMBIC, "upgrade", "head"], + [*UV_ALEMBIC, "current", "--check-heads"], + ] + assert "secret" not in result.stdout + result.stderr + + +def test_verify_mode_only_requires_the_database_at_the_head(fakes): + fakes.add("uv") + + result = _run_migration_script(fakes, SIMULATION_MIGRATION_MODE="verify") + + assert result.returncode == 0, result.stderr + assert fakes.calls("uv") == [[*UV_ALEMBIC, "current", "--check-heads"]] + + +@pytest.mark.parametrize("mode", ["", "downgrade", "UPGRADE"]) +def test_the_migration_script_refuses_an_unknown_mode(fakes, mode): + fakes.add("uv") + + result = _run_migration_script(fakes, SIMULATION_MIGRATION_MODE=mode) + + assert result.returncode != 0 + assert "SIMULATION_MIGRATION_MODE" in result.stderr + assert fakes.calls("uv") == [] + + +def test_the_migration_script_refuses_to_run_without_the_migrator_url(fakes): + fakes.add("uv") + + result = _run_migration_script(fakes, SIMULATION_MIGRATION_DATABASE_URL="") + + assert result.returncode != 0 + assert MIGRATION_SECRET in result.stderr + assert fakes.calls("uv") == [] + + +def test_a_failed_upgrade_fails_the_job_before_the_head_check(fakes): + fakes.add("uv", 'if [[ " $* " == *" upgrade "* ]]; then exit 3; fi\n') + + result = _run_migration_script(fakes) + + assert result.returncode == 3 + assert len(fakes.calls("uv")) == 1 + + +def test_only_the_default_branch_applies_migrations(): + migration = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)[MIGRATION_JOB] + steps = {step.get("name"): step for step in migration["steps"]} + mode = steps["Apply simulation schema migrations"]["env"][ + "SIMULATION_MIGRATION_MODE" + ] + + # A deployment dispatched from another branch must not apply that branch's + # revisions; it only requires the database to be at its chain's head. + assert ( + evaluate_github_expression(mode, {"github": {"ref": "refs/heads/main"}}) + == "upgrade" + ) + assert ( + evaluate_github_expression(mode, {"github": {"ref": "refs/heads/feature"}}) + == "verify" + ) + + +# --- The entrypoint's runtime URL ----------------------------------------- + + +def _run_deploy(tmp_path, **overrides): + environment = cloud_run_deploy_environment(tmp_path) | overrides + result = subprocess.run( + [CLOUD_RUN_DEPLOY_SCRIPT], + check=False, + capture_output=True, + text=True, + env=environment, + ) + return result, environment + + +def _secrets_argument(environment) -> str: + arguments = ( + Path(environment["GCLOUD_ARGUMENTS_CAPTURE"]) + .read_text(encoding="utf-8") + .splitlines() + ) + return arguments[arguments.index("--set-secrets") + 1] + + +@pytest.mark.parametrize("deploy_stage12", ["true", "false"]) +def test_the_entrypoint_receives_the_runtime_url_as_a_pinned_secret( + tmp_path, deploy_stage12 +): + result, environment = _run_deploy(tmp_path, DEPLOY_STAGE12_V2=deploy_stage12) + + assert result.returncode == 0, result.stderr + assert ( + "SIMULATION_DATABASE_URL=simulation-staging-runtime-database-url:3" + in _secrets_argument(environment).split(",") + ) + runtime_environment = json.loads( + Path(environment["GCLOUD_ENV_FILE_CAPTURE"]).read_text(encoding="utf-8") + ) + assert "SIMULATION_DATABASE_URL" not in runtime_environment + assert MIGRATION_SECRET not in runtime_environment + + +@pytest.mark.parametrize( + "secret_name", + ["", "not a secret name", "projects/x/secrets/name/versions/1"], +) +def test_the_entrypoint_deploy_requires_a_valid_runtime_url_secret( + tmp_path, secret_name +): + result, _ = _run_deploy(tmp_path, SIMULATION_DATABASE_URL_SECRET_NAME=secret_name) + + assert result.returncode != 0 + assert "SIMULATION_DATABASE_URL_SECRET_NAME" in result.stderr or ( + "Secret Manager reference" in result.stderr + ) + + +def test_the_deploy_step_passes_the_runtime_secret_name(): + deploy_entrypoint = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["deploy_entrypoint"] + deploy_step = next( + step + for step in deploy_entrypoint["steps"] + if step.get("name") == "Deploy Cloud Run entrypoint candidate" + ) + + assert deploy_step["env"]["SIMULATION_DATABASE_URL_SECRET_NAME"] == ( + "${{ vars.SIMULATION_DATABASE_URL_SECRET_NAME }}" + ) + + +# --- The candidate smoke -------------------------------------------------- + + +def test_the_candidate_smoke_requires_the_current_database_revision(): + integration = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["integration"] + smoke = next( + step + for step in integration["steps"] + if step.get("name") == "Verify the complete deployed request path" + ) + + assert smoke["env"]["REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION"] == "true" + + +def _ready_body(revision, expected_revision="0001"): + return json.dumps( + { + "status": "ready", + "simulation_database": { + "reachable": revision is not None, + "revision": revision, + "expected_revision": expected_revision, + }, + } + ) + + +def _run_smoke(tmp_path, ready_body, *, require): + state_file, revisions_file = write_fake_smoke_curl(tmp_path, ["revision-1"]) + environment = smoke_test_env(tmp_path, state_file, revisions_file) | { + "FAKE_CURL_READY": ready_body, + } + if require is not None: + environment["REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION"] = require + return subprocess.run( + ["bash", SMOKE_SCRIPT, "https://candidate.example.test", "revision-1"], + check=False, + capture_output=True, + text=True, + env=environment, + ) + + +def test_the_smoke_accepts_a_database_at_the_expected_revision(tmp_path): + result = _run_smoke(tmp_path, _ready_body("0001"), require="true") + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + "ready_body", + [ + _ready_body(None), + _ready_body("0001", expected_revision="0002"), + json.dumps({"status": "ready"}), + ], + ids=["unreachable", "behind", "unconfigured"], +) +def test_the_smoke_rejects_a_database_not_at_the_expected_revision( + tmp_path, ready_body +): + result = _run_smoke(tmp_path, ready_body, require="true") + + assert result.returncode == 1 + assert "simulation database" in result.stderr + + +def _run_smoke_with_ready_bodies(tmp_path, bodies): + bodies_file = tmp_path / "ready-bodies" + bodies_file.write_text("\n".join(bodies) + "\n", encoding="utf-8") + state_file, revisions_file = write_fake_smoke_curl(tmp_path, ["revision-1"]) + environment = smoke_test_env(tmp_path, state_file, revisions_file) | { + "FAKE_CURL_READY_BODIES": str(bodies_file), + "REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION": "true", + "SIMULATION_DATABASE_READY_MAX_ATTEMPTS": "3", + } + result = subprocess.run( + ["bash", SMOKE_SCRIPT, "https://candidate.example.test", "revision-1"], + check=False, + capture_output=True, + text=True, + env=environment, + ) + ready_requests = int(Path(f"{bodies_file}.count").read_text(encoding="utf-8")) + return result, ready_requests + + +def test_the_smoke_waits_for_a_database_a_cold_instance_has_not_reached(tmp_path): + result, ready_requests = _run_smoke_with_ready_bodies( + tmp_path, [_ready_body(None), _ready_body(None), _ready_body("0001")] + ) + + assert result.returncode == 0, result.stderr + assert ready_requests == 3 + + +def test_the_smoke_stops_waiting_for_an_unreachable_database(tmp_path): + result, ready_requests = _run_smoke_with_ready_bodies(tmp_path, [_ready_body(None)]) + + assert result.returncode == 1 + assert ready_requests == 3 + assert "simulation database" in result.stderr + + +def test_the_smoke_does_not_wait_for_a_database_at_another_revision(tmp_path): + result, ready_requests = _run_smoke_with_ready_bodies( + tmp_path, [_ready_body("0001", expected_revision="0002"), _ready_body("0002")] + ) + + assert result.returncode == 1 + assert ready_requests == 1 + + +def test_stable_and_rollback_smokes_ignore_the_database_revision(tmp_path): + result = _run_smoke(tmp_path, _ready_body(None), require=None) + + assert result.returncode == 0, result.stderr + + +def test_the_smoke_refuses_an_unknown_database_requirement(tmp_path): + result = _run_smoke(tmp_path, _ready_body("0001"), require="yes") + + assert result.returncode == 2 + assert "REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION" in result.stderr diff --git a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py index daf0a01c1..25e967219 100644 --- a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py +++ b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py @@ -2,7 +2,9 @@ DDL_PATTERN is shared with sim.boundaries.sqlalchemy-and-alembic-only, which matches it against the string literals of every source directory; see -database_access_support.py. +database_access_support.py. The Simulation Entrypoint's own schema has its own +ownership rules, checked in +libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py. """ from __future__ import annotations @@ -41,12 +43,6 @@ def test_rule_proof_stage12_no_schema_ddl() -> None: assert DDL_PATTERN.search(good) is None -def test_simulation_repository_has_no_schema_migration_chain() -> None: - assert not list(REPOSITORY_ROOT.glob("alembic*.ini")) - assert not list(REPOSITORY_ROOT.rglob("alembic/versions")) - assert not list(REPOSITORY_ROOT.rglob("migrations/versions")) - - def test_stage12_runtime_cannot_create_production_user_associations() -> None: forbidden = { "user_simulation", diff --git a/projects/policyengine-simulation-entry/uv.lock b/projects/policyengine-simulation-entry/uv.lock index c3bfa7d53..31a2409dd 100644 --- a/projects/policyengine-simulation-entry/uv.lock +++ b/projects/policyengine-simulation-entry/uv.lock @@ -901,6 +901,7 @@ dependencies = [ { name = "policyengine-observability", extra = ["fastapi", "google", "httpx", "otlp-grpc"] }, { name = "policyengine-simulation-contract" }, { name = "policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence" }, { name = "pyjwt" }, { name = "uvicorn" }, @@ -928,6 +929,7 @@ requires-dist = [ { name = "policyengine-observability", extras = ["fastapi", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.4,<4" }, { name = "policyengine-simulation-contract", editable = "../../libs/policyengine-simulation-contract" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence", editable = "../../libs/policyengine-stage12-persistence" }, { name = "pyjwt", specifier = ">=2.10.1,<3" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, @@ -963,6 +965,28 @@ requires-dist = [ ] provides-extras = ["test", "build"] +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "../../libs/policyengine-simulation-persistence" } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", marker = "extra == 'migrations'", specifier = ">=1.17.1,<2" }, + { name = "alembic", marker = "extra == 'test'", specifier = ">=1.17.1,<2" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<3" }, +] +provides-extras = ["migrations", "test", "build"] + [[package]] name = "policyengine-stage12-persistence" version = "0.1.0" diff --git a/ruff.toml b/ruff.toml index 38c26d845..6cc798b35 100644 --- a/ruff.toml +++ b/ruff.toml @@ -9,7 +9,8 @@ per-file-target-version = { "scripts/**" = "py311", "projects/policyengine-simulation-executor/**" = "py313" } # Directories that are not regular packages on purpose, so they have no # __init__.py: standalone scripts, the implicit namespace package that holds -# scripts.rules, and proof fixtures. Ruff resolves modules as if each listed +# scripts.rules, proof fixtures, and the Alembic chain, whose environment and +# revisions Alembic loads by path. Ruff resolves modules as if each listed # directory, and every directory under it, had an __init__.py. This is a # top-level setting: ruff rejects it in [lint]. The executor's src/modal/utils # scripts are not listed, because ruff would then treat them as part of a @@ -17,6 +18,7 @@ per-file-target-version = { "scripts/**" = "py311", "projects/policyengine-simul # an inline INP001 suppression instead. namespace-packages = [ ".github/scripts", + "migrations", "projects/policyengine-simulation-executor/scripts", "rules/fixtures", "scripts", diff --git a/rules/fixtures/sim.deploy.container-context-minimal/bad.dockerignore b/rules/fixtures/sim.deploy.container-context-minimal/bad.dockerignore new file mode 100644 index 000000000..d8ed8cfbb --- /dev/null +++ b/rules/fixtures/sim.deploy.container-context-minimal/bad.dockerignore @@ -0,0 +1,4 @@ +# Proof fixture: the context leaves out one library the project installs. +libs/* +!libs/fixture-contract/ +!libs/fixture-contract/** diff --git a/rules/fixtures/sim.deploy.container-context-minimal/good.dockerignore b/rules/fixtures/sim.deploy.container-context-minimal/good.dockerignore new file mode 100644 index 000000000..d80779a02 --- /dev/null +++ b/rules/fixtures/sim.deploy.container-context-minimal/good.dockerignore @@ -0,0 +1,6 @@ +# Proof fixture: the context includes every library the project installs. +libs/* +!libs/fixture-contract/ +!libs/fixture-contract/** +!libs/fixture-persistence/ +!libs/fixture-persistence/** diff --git a/rules/fixtures/sim.deploy.container-context-minimal/pyproject.toml b/rules/fixtures/sim.deploy.container-context-minimal/pyproject.toml new file mode 100644 index 000000000..aa2c53cc7 --- /dev/null +++ b/rules/fixtures/sim.deploy.container-context-minimal/pyproject.toml @@ -0,0 +1,9 @@ +# Proof fixture: a project that installs two local libraries by path. +[project] +name = "fixture-entrypoint" +version = "0.1.0" +dependencies = ["fixture-contract", "fixture-persistence"] + +[tool.uv.sources] +fixture-contract = { path = "../../libs/fixture-contract", editable = true } +fixture-persistence = { path = "../../libs/fixture-persistence", editable = true } diff --git a/rules/fixtures/sim.deploy.workflow-shell-safety/bad_deploy_script.sh b/rules/fixtures/sim.deploy.workflow-shell-safety/bad_deploy_script.sh new file mode 100644 index 000000000..73d926095 --- /dev/null +++ b/rules/fixtures/sim.deploy.workflow-shell-safety/bad_deploy_script.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Proof fixture: one array assignment resolves two secrets, so a failure to +# resolve the first is ignored and the deploy goes on with an empty reference. +set -euo pipefail + +runtime_secrets=( + "OLD_GATEWAY_AUTH_CLIENT_SECRET=$(resolve_secret_reference "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}")" + "SIMULATION_DATABASE_URL=$(resolve_secret_reference "${SIMULATION_DATABASE_URL_SECRET_NAME}")" +) diff --git a/rules/fixtures/sim.deploy.workflow-shell-safety/good_deploy_script.sh b/rules/fixtures/sim.deploy.workflow-shell-safety/good_deploy_script.sh new file mode 100644 index 000000000..6edd73590 --- /dev/null +++ b/rules/fixtures/sim.deploy.workflow-shell-safety/good_deploy_script.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Proof fixture: each secret is resolved in its own assignment, so set -e stops +# the script at the first failure. +set -euo pipefail + +runtime_secrets=() + +add_runtime_secret() { + local variable_name="$1" + local resolved_reference + resolved_reference="$(resolve_secret_reference "$2")" + runtime_secrets+=("${variable_name}=${resolved_reference}") +} + +add_runtime_secret OLD_GATEWAY_AUTH_CLIENT_SECRET "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}" +add_runtime_secret SIMULATION_DATABASE_URL "${SIMULATION_DATABASE_URL_SECRET_NAME}" diff --git a/rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml b/rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml new file mode 100644 index 000000000..cb1ae0f5a --- /dev/null +++ b/rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml @@ -0,0 +1,7 @@ +# Proof fixture: a Modal project that depends on the persistence library. +[project] +name = "policyengine-simulation-executor" +dependencies = [ + "policyengine-simulation-contract", + "policyengine-simulation-persistence", +] diff --git a/rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml b/rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml new file mode 100644 index 000000000..b75b2ff09 --- /dev/null +++ b/rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml @@ -0,0 +1,6 @@ +# Proof fixture: a Modal project that depends only on the contract library. +[project] +name = "policyengine-simulation-executor" +dependencies = [ + "policyengine-simulation-contract", +] diff --git a/rules/fixtures/sim.persistence.migrate-before-entrypoint/bad_workflow.yml b/rules/fixtures/sim.persistence.migrate-before-entrypoint/bad_workflow.yml new file mode 100644 index 000000000..f6f66bd65 --- /dev/null +++ b/rules/fixtures/sim.persistence.migrate-before-entrypoint/bad_workflow.yml @@ -0,0 +1,19 @@ +# Proof fixture: the entrypoint deploys without waiting for the migration, +# the migrator URL reaches every step of the job, and revisions are applied +# from whatever branch dispatched the deployment. +name: Bad deployment +on: workflow_dispatch +jobs: + migrate_simulation_database: + runs-on: ubuntu-latest + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + steps: + - name: Apply simulation schema migrations + env: + SIMULATION_MIGRATION_MODE: upgrade + run: ../../.github/scripts/migrate-simulation-database.sh + deploy_entrypoint: + runs-on: ubuntu-latest + steps: + - run: .github/scripts/deploy-cloud-run-simulation-entry.sh diff --git a/rules/fixtures/sim.persistence.migrate-before-entrypoint/good_workflow.yml b/rules/fixtures/sim.persistence.migrate-before-entrypoint/good_workflow.yml new file mode 100644 index 000000000..c45ddcaf2 --- /dev/null +++ b/rules/fixtures/sim.persistence.migrate-before-entrypoint/good_workflow.yml @@ -0,0 +1,20 @@ +# Proof fixture: the migration job alone receives the migrator URL, in its +# migration step; it applies revisions only from main; and the entrypoint +# deploys only after it succeeded. +name: Good deployment +on: workflow_dispatch +jobs: + migrate_simulation_database: + runs-on: ubuntu-latest + steps: + - name: Apply simulation schema migrations + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + SIMULATION_MIGRATION_MODE: ${{ github.ref == 'refs/heads/main' && 'upgrade' || 'verify' }} + run: ../../.github/scripts/migrate-simulation-database.sh + deploy_entrypoint: + if: ${{ always() && needs.migrate_simulation_database.result == 'success' }} + needs: [migrate_simulation_database] + runs-on: ubuntu-latest + steps: + - run: .github/scripts/deploy-cloud-run-simulation-entry.sh diff --git a/rules/fixtures/sim.persistence.migration-cycle/bad_chain/env.py b/rules/fixtures/sim.persistence.migration-cycle/bad_chain/env.py new file mode 100644 index 000000000..c9aa31716 --- /dev/null +++ b/rules/fixtures/sim.persistence.migration-cycle/bad_chain/env.py @@ -0,0 +1,20 @@ +"""Proof fixture environment: applies this chain to the migration database.""" + +import os + +from alembic import context +from sqlalchemy import create_engine + +url = os.environ["SIMULATION_MIGRATION_DATABASE_URL"].replace( + "postgresql://", "postgresql+psycopg://", 1 +) +engine = create_engine(url) +with engine.begin() as connection: + context.configure( + connection=connection, + target_metadata=None, + version_table="proof_bad_chain_version", + ) + with context.begin_transaction(): + context.run_migrations() +engine.dispose() diff --git a/rules/fixtures/sim.persistence.migration-cycle/bad_chain/versions/0001_proof_table.py b/rules/fixtures/sim.persistence.migration-cycle/bad_chain/versions/0001_proof_table.py new file mode 100644 index 000000000..7b0dde780 --- /dev/null +++ b/rules/fixtures/sim.persistence.migration-cycle/bad_chain/versions/0001_proof_table.py @@ -0,0 +1,21 @@ +"""Proof fixture: a revision whose downgrade leaves its table behind. + +Upgrading again after the downgrade fails, because the table still exists. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0001" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Create the proof table.""" + op.create_table("proof_bad_table", sa.Column("id", sa.Integer(), primary_key=True)) + + +def downgrade() -> None: + """Leave the proof table behind, which the cycle must catch.""" diff --git a/rules/fixtures/sim.persistence.migration-cycle/good_chain/env.py b/rules/fixtures/sim.persistence.migration-cycle/good_chain/env.py new file mode 100644 index 000000000..f2a7f3950 --- /dev/null +++ b/rules/fixtures/sim.persistence.migration-cycle/good_chain/env.py @@ -0,0 +1,20 @@ +"""Proof fixture environment: applies this chain to the migration database.""" + +import os + +from alembic import context +from sqlalchemy import create_engine + +url = os.environ["SIMULATION_MIGRATION_DATABASE_URL"].replace( + "postgresql://", "postgresql+psycopg://", 1 +) +engine = create_engine(url) +with engine.begin() as connection: + context.configure( + connection=connection, + target_metadata=None, + version_table="proof_good_chain_version", + ) + with context.begin_transaction(): + context.run_migrations() +engine.dispose() diff --git a/rules/fixtures/sim.persistence.migration-cycle/good_chain/versions/0001_proof_table.py b/rules/fixtures/sim.persistence.migration-cycle/good_chain/versions/0001_proof_table.py new file mode 100644 index 000000000..06869afa1 --- /dev/null +++ b/rules/fixtures/sim.persistence.migration-cycle/good_chain/versions/0001_proof_table.py @@ -0,0 +1,19 @@ +"""Proof fixture: a revision whose downgrade undoes its upgrade.""" + +import sqlalchemy as sa +from alembic import op + +revision = "0001" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Create the proof table.""" + op.create_table("proof_good_table", sa.Column("id", sa.Integer(), primary_key=True)) + + +def downgrade() -> None: + """Drop the proof table.""" + op.drop_table("proof_good_table") diff --git a/rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml b/rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml new file mode 100644 index 000000000..739dbc28b --- /dev/null +++ b/rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml @@ -0,0 +1,11 @@ +# Proof fixture: a step that copies the simulation database URL into a Modal secret. +name: Bad deployment +on: workflow_dispatch +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Synchronize Modal runtime secrets + env: + SIMULATION_DATABASE_URL: ${{ secrets.SIMULATION_DATABASE_URL }} + run: uv run modal secret create simulation-database "SIMULATION_DATABASE_URL=${SIMULATION_DATABASE_URL}" --force diff --git a/rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml b/rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml new file mode 100644 index 000000000..9a2318ce2 --- /dev/null +++ b/rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml @@ -0,0 +1,16 @@ +# Proof fixture: Modal receives only the Stage 12 secret name; the simulation +# migration URL reaches the migration step alone. +name: Good deployment +on: workflow_dispatch +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Synchronize the separate Stage 12 runtime secret + env: + STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }} + run: uv run modal secret create stage12-runtime "STAGE12_DATABASE_URL_SECRET_NAME=${STAGE12_DATABASE_URL_SECRET_NAME}" --force + - name: Apply simulation schema migrations + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + run: .github/scripts/simulation-database-migrate.sh diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/alembic.ini b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/alembic.ini new file mode 100644 index 000000000..6cb09e697 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = %(here)s diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/env.py b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/env.py new file mode 100644 index 000000000..ab2608882 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/env.py @@ -0,0 +1,7 @@ +"""Proof fixture: an Alembic environment that versions the simulation schema.""" + +from alembic import context + +SIMULATION_SCHEMA = "simulation" + +context.configure(version_table_schema=SIMULATION_SCHEMA) diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/versions/0001_initial.py b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/versions/0001_initial.py new file mode 100644 index 000000000..5d6301f03 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/versions/0001_initial.py @@ -0,0 +1,4 @@ +"""Proof fixture: the chain's first revision.""" + +revision = "0001" +down_revision = None diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/alembic.ini b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/alembic.ini new file mode 100644 index 000000000..6f4ed3585 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = %(here)s/migrations diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/migrations/versions/0001_service.py b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/migrations/versions/0001_service.py new file mode 100644 index 000000000..3a3f03b05 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/migrations/versions/0001_service.py @@ -0,0 +1,4 @@ +"""Proof fixture: a revision in a second, forbidden chain.""" + +revision = "0001" +down_revision = None diff --git a/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/alembic.ini b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/alembic.ini new file mode 100644 index 000000000..6cb09e697 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = %(here)s diff --git a/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/env.py b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/env.py new file mode 100644 index 000000000..ab2608882 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/env.py @@ -0,0 +1,7 @@ +"""Proof fixture: an Alembic environment that versions the simulation schema.""" + +from alembic import context + +SIMULATION_SCHEMA = "simulation" + +context.configure(version_table_schema=SIMULATION_SCHEMA) diff --git a/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/versions/0001_initial.py b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/versions/0001_initial.py new file mode 100644 index 000000000..5d6301f03 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/versions/0001_initial.py @@ -0,0 +1,4 @@ +"""Proof fixture: the chain's first revision.""" + +revision = "0001" +down_revision = None diff --git a/rules/registry.toml b/rules/registry.toml index f298eb988..ea16e6a58 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -21,6 +21,10 @@ title = "Service and package boundaries" id = "stage12" title = "Stage 12 persistence" +[[category]] +id = "persistence" +title = "Simulation persistence" + [[category]] id = "deployment" title = "Deployment pipeline" @@ -112,7 +116,7 @@ id = "sim.boundaries.sqlalchemy-and-alembic-only" category = "boundaries" statement = "Code connects to databases only through SQLAlchemy engines and connections, never by importing a database driver directly, and runtime code never changes the schema, which changes only through Alembic migrations." rationale = "docs/engineering/skills/database-access.md#rules" -scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py", ".github/scripts/**/*.py"] +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py", ".github/scripts/**/*.py", "migrations/**/*.py"] tier = "mechanical" status = "enforced" autofix = false @@ -133,48 +137,165 @@ good_fixture = "rules/fixtures/sim.boundaries.sqlalchemy-and-alembic-only/good_d project = "projects/policyengine-simulation-entry" test = "tests/test_database_access_boundaries.py::test_rule_proof_sqlalchemy_and_alembic_only" -# --- Stage 12 persistence -------------------------------------------------- +# --- Simulation persistence ----------------------------------------------- [[rule]] -id = "sim.stage12.no-schema-ddl" -category = "stage12" -statement = "Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema." -rationale = "libs/policyengine-stage12-persistence/README.md" -scope = ["libs/policyengine-stage12-persistence/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] +id = "sim.persistence.one-migration-chain" +category = "persistence" +statement = "The simulation schema changes only through the one Alembic chain at migrations/, configured by migrations/alembic.ini and versioned in simulation.alembic_version; no other Alembic configuration, environment or revision directory exists." +rationale = "libs/policyengine-simulation-persistence/README.md#ownership-rules" +scope = ["migrations/env.py", "migrations/alembic.ini", "migrations/versions/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = ["tests/test_ownership_boundaries.py::test_repository_has_one_simulation_migration_chain"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.one-migration-chain/bad_repository" +good_fixture = "rules/fixtures/sim.persistence.one-migration-chain/good_repository" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_ownership_boundaries.py::test_rule_proof_one_migration_chain" + +[[rule]] +id = "sim.persistence.migration-cycle" +category = "persistence" +statement = "On a fresh PostgreSQL, the simulation chain upgrades to its head, passes alembic check, downgrades to base and upgrades again, run by a non-superuser migrator, and the runtime role can read the revision but cannot change the schema." +rationale = "libs/policyengine-simulation-persistence/README.md#tests" +scope = ["migrations/versions/*.py", "libs/policyengine-simulation-persistence/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = [ + "tests/test_migration_cycle.py::test_upgrade_check_downgrade_upgrade_cycle", + "tests/test_migration_cycle.py::test_the_version_table_lives_only_in_the_simulation_schema", + "tests/test_migration_cycle.py::test_the_runtime_role_reads_the_revision_but_cannot_change_the_schema", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "simulation-database" +invocation = "uv run --frozen pytest tests/ -v -m postgres" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.migration-cycle/bad_chain/versions/0001_proof_table.py" +good_fixture = "rules/fixtures/sim.persistence.migration-cycle/good_chain/versions/0001_proof_table.py" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_migration_cycle.py::test_rule_proof_migration_cycle" + +[[rule]] +id = "sim.persistence.executor-independent" +category = "persistence" +statement = "The Modal executor and gateway never declare, lock or import policyengine-simulation-persistence; only the Simulation Entrypoint and the migration job install it." +rationale = "libs/policyengine-simulation-persistence/README.md#ownership-rules" +scope = ["projects/policyengine-simulation-executor/pyproject.toml", "projects/policyengine-simulation-gateway/pyproject.toml", "projects/policyengine-simulation-executor/src/**/*.py", "projects/policyengine-simulation-gateway/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = ["tests/test_ownership_boundaries.py::test_modal_projects_never_depend_on_the_persistence_library"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml" +good_fixture = "rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_ownership_boundaries.py::test_rule_proof_executor_independent" + +[[rule]] +id = "sim.persistence.no-modal-database-credentials" +category = "persistence" +statement = "No Modal secret, Modal deployment or Modal application receives a database URL other than the comparison path's own STAGE12_DATABASE_URL, which Modal may hold until the comparison path is removed; in particular never SIMULATION_DATABASE_URL or SIMULATION_MIGRATION_DATABASE_URL." +rationale = "libs/policyengine-simulation-persistence/README.md#ownership-rules" +scope = [".github/workflows/*.yml", ".github/scripts/*.sh", "projects/policyengine-simulation-executor/src/**/*.py", "projects/policyengine-simulation-gateway/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = [ + "tests/test_ownership_boundaries.py::test_modal_receives_no_database_url_but_the_comparison_paths", + "tests/test_ownership_boundaries.py::test_modal_steps_may_carry_only_the_comparison_paths_database_url", + "tests/test_ownership_boundaries.py::test_the_comparison_paths_database_url_may_reach_modal_until_it_is_removed", + "tests/test_ownership_boundaries.py::test_a_modal_script_may_not_carry_another_database_url", + "tests/test_ownership_boundaries.py::test_modal_sources_may_name_only_the_comparison_paths_database_url", + "tests/test_ownership_boundaries.py::test_a_temporary_modal_app_may_not_receive_another_database_url", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml" +good_fixture = "rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_ownership_boundaries.py::test_rule_proof_no_modal_database_credentials" + +[[rule]] +id = "sim.persistence.migrate-before-entrypoint" +category = "persistence" +statement = "Each deployment migrates the simulation schema in its own job, the only one that receives SIMULATION_MIGRATION_DATABASE_URL, before the entrypoint candidate deploys, applying revisions only from main and otherwise requiring the database at the chain's head; the entrypoint receives SIMULATION_DATABASE_URL as a pinned Secret Manager version; and the candidate smoke requires the database at the image's revision." +rationale = "libs/policyengine-simulation-persistence/README.md#deployment" +scope = [".github/workflows/simulation-deploy.reusable.yml", ".github/scripts/migrate-simulation-database.sh", ".github/scripts/deploy-cloud-run-simulation-entry.sh", ".github/scripts/cloud-run-simulation-entry-smoke.sh"] tier = "mechanical" status = "enforced" autofix = false [rule.enforcer] kind = "pytest" project = "projects/policyengine-simulation-entry" -nodes = ["tests/test_stage12_schema_authority.py::test_simulation_runtimes_contain_no_schema_ddl"] +nodes = [ + "tests/test_simulation_database_deployment.py::test_the_deploy_workflow_follows_the_migration_wiring", + "tests/test_simulation_database_deployment.py::test_the_migration_job_runs_before_the_entrypoint_deploys", + "tests/test_simulation_database_deployment.py::test_only_the_migration_step_receives_the_migrator_url", + "tests/test_simulation_database_deployment.py::test_only_the_default_branch_applies_migrations", + "tests/test_simulation_database_deployment.py::test_the_entrypoint_receives_the_runtime_url_as_a_pinned_secret", + "tests/test_simulation_database_deployment.py::test_the_candidate_smoke_requires_the_current_database_revision", +] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run --frozen pytest tests/" [rule.proof] -bad_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py" -good_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py" +bad_fixture = "rules/fixtures/sim.persistence.migrate-before-entrypoint/bad_workflow.yml" +good_fixture = "rules/fixtures/sim.persistence.migrate-before-entrypoint/good_workflow.yml" project = "projects/policyengine-simulation-entry" -test = "tests/test_stage12_schema_authority.py::test_rule_proof_stage12_no_schema_ddl" +test = "tests/test_simulation_database_deployment.py::test_rule_proof_migrate_before_entrypoint" + +# --- Stage 12 persistence -------------------------------------------------- [[rule]] -id = "sim.stage12.no-migration-chain" +id = "sim.stage12.no-schema-ddl" category = "stage12" -statement = "This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api." +statement = "Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema." rationale = "libs/policyengine-stage12-persistence/README.md" -scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +scope = ["libs/policyengine-stage12-persistence/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] tier = "mechanical" status = "enforced" autofix = false [rule.enforcer] kind = "pytest" project = "projects/policyengine-simulation-entry" -nodes = ["tests/test_stage12_schema_authority.py::test_simulation_repository_has_no_schema_migration_chain"] +nodes = ["tests/test_stage12_schema_authority.py::test_simulation_runtimes_contain_no_schema_ddl"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py" +good_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py" +project = "projects/policyengine-simulation-entry" +test = "tests/test_stage12_schema_authority.py::test_rule_proof_stage12_no_schema_ddl" [[rule]] id = "sim.stage12.no-production-user-associations" @@ -317,9 +438,9 @@ invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.workflow-shell-safety" category = "deployment" -statement = "Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions." +statement = "Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, pins secret versions, stops before deploying when any runtime secret cannot be resolved, and resolves the Stage 12 secrets with the Stage 12 block's own helper." rationale = "docs/operations" -scope = [".github/workflows/*.yml"] +scope = [".github/workflows/*.yml", ".github/scripts/deploy-cloud-run-simulation-entry.sh"] tier = "mechanical" status = "enforced" autofix = false @@ -329,11 +450,19 @@ project = "projects/policyengine-simulation-entry" nodes = [ "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_does_not_interpolate_github_values_in_shell", "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_escapes_environment_and_pins_secret_versions", + "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_stops_when_a_runtime_secret_cannot_be_resolved", + "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_resolves_each_secret_in_its_own_assignment", + "tests/test_deployment_assets_cloud_run.py::test_stage12_secrets_are_resolved_by_the_stage12_block_alone", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.deploy.workflow-shell-safety/bad_deploy_script.sh" +good_fixture = "rules/fixtures/sim.deploy.workflow-shell-safety/good_deploy_script.sh" +project = "projects/policyengine-simulation-entry" +test = "tests/test_deployment_assets_cloud_run.py::test_rule_proof_workflow_shell_safety" [[rule]] id = "sim.deploy.revision-tags-valid" @@ -533,9 +662,9 @@ invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.container-context-minimal" category = "deployment" -statement = "Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform." +statement = "Container builds exclude local environments and unrelated projects but include every local library the entrypoint installs, and the entrypoint container builds for the requested target platform." rationale = "docs/operations" -scope = ["projects/policyengine-simulation-entry/Dockerfile"] +scope = ["projects/policyengine-simulation-entry/Dockerfile", "projects/policyengine-simulation-entry/Dockerfile.dockerignore"] tier = "mechanical" status = "enforced" autofix = false @@ -544,12 +673,18 @@ kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ "tests/test_deployment_assets_traffic.py::test_container_context_excludes_local_environments_and_unrelated_projects", + "tests/test_deployment_assets_traffic.py::test_container_context_includes_every_local_library_the_entrypoint_installs", "tests/test_deployment_assets_cloud_run.py::test_entrypoint_container_uses_requested_target_platform", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.deploy.container-context-minimal/bad.dockerignore" +good_fixture = "rules/fixtures/sim.deploy.container-context-minimal/good.dockerignore" +project = "projects/policyengine-simulation-entry" +test = "tests/test_deployment_assets_traffic.py::test_rule_proof_container_context_minimal" [[rule]] id = "sim.deploy.docker-images-build" @@ -1067,11 +1202,12 @@ invocation = "unittest discover -s scripts/rules/tests" [[rule]] id = "sim.quality.pyright-libraries-and-entry" category = "quality" -statement = "The contract, observability, Stage 12 persistence and fastapi libraries and the simulation entry have no Pyright errors, checked with the pinned Pyright version in each package's own locked environment." +statement = "The contract, observability, simulation persistence, Stage 12 persistence and fastapi libraries and the simulation entry have no Pyright errors, checked with the pinned Pyright version in each package's own locked environment." rationale = "docs/engineering/skills/type-checking.md#rules" scope = [ "libs/policyengine-simulation-contract/src/**/*.py", "libs/policyengine-simulation-observability/src/**/*.py", + "libs/policyengine-simulation-persistence/src/**/*.py", "libs/policyengine-stage12-persistence/src/**/*.py", "libs/policyengine-fastapi/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", @@ -1084,6 +1220,7 @@ kind = "pyright" projects = [ "libs/policyengine-simulation-contract", "libs/policyengine-simulation-observability", + "libs/policyengine-simulation-persistence", "libs/policyengine-stage12-persistence", "libs/policyengine-fastapi", "projects/policyengine-simulation-entry",