diff --git a/.github/scripts/check-uv-locks.sh b/.github/scripts/check-uv-locks.sh new file mode 100755 index 000000000..9f6b453e6 --- /dev/null +++ b/.github/scripts/check-uv-locks.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Check that every project's uv.lock matches its pyproject.toml. +# +# The projects are the directories of the tracked pyproject.toml files (test +# fixtures excluded), so a new project is checked without editing this script +# or the workflow. Every project is checked before the script fails, so one run +# reports every stale lock. See docs/engineering/skills/dependencies.md. +set -euo pipefail + +failed=0 +while IFS= read -r pyproject; do + project="$(dirname "${pyproject}")" + echo "Checking ${project}/uv.lock" + if ! uv lock --check --project "${project}"; then + echo "::error file=${project}/uv.lock::uv.lock does not match pyproject.toml; run uv lock --project ${project}" >&2 + failed=1 + fi +done < <(git ls-files -- '*pyproject.toml' ':!:**/fixtures/**') + +exit "${failed}" diff --git a/.github/scripts/modal-extract-versions.sh b/.github/scripts/modal-extract-versions.sh index 8f2bc2699..254d5ae0b 100755 --- a/.github/scripts/modal-extract-versions.sh +++ b/.github/scripts/modal-extract-versions.sh @@ -12,4 +12,4 @@ PROJECT_DIR="${1:-.}" cd "$PROJECT_DIR" -uv run python -m src.modal.utils.extract_bundle_versions +uv run --frozen python -m src.modal.utils.extract_bundle_versions diff --git a/.github/scripts/modal-image-smoke.sh b/.github/scripts/modal-image-smoke.sh index ea9b092d0..dc9c0062f 100755 --- a/.github/scripts/modal-image-smoke.sh +++ b/.github/scripts/modal-image-smoke.sh @@ -21,7 +21,7 @@ fi # PolicyEngine Core reads the HUGGING_FACE_TOKEN compatibility name in Modal. ( cd "$REPO_ROOT/projects/policyengine-simulation-executor" - uv run modal secret create pe-uk-private-hf-read-token \ + uv run --frozen modal secret create pe-uk-private-hf-read-token \ "HUGGING_FACE_TOKEN=$PE_UK_PRIVATE_HF_READ_TOKEN" \ --env="$MODAL_ENV" \ --force @@ -30,20 +30,20 @@ fi echo "=== Gateway image smoke (env: $MODAL_ENV) ===" ( cd "$REPO_ROOT/projects/policyengine-simulation-gateway" - uv run modal run --env="$MODAL_ENV" \ + uv run --frozen modal run --env="$MODAL_ENV" \ src/policyengine_simulation_gateway/smoke_app.py ) echo "=== Executor image smoke (env: $MODAL_ENV) ===" ( cd "$REPO_ROOT/projects/policyengine-simulation-executor" - uv run modal run --env="$MODAL_ENV" src/modal/smoke_app.py + uv run --frozen modal run --env="$MODAL_ENV" src/modal/smoke_app.py ) echo "=== Stage 12 static runtime file smoke (env: $MODAL_ENV) ===" ( cd "$REPO_ROOT/projects/policyengine-simulation-executor" - uv run modal run --env="$MODAL_ENV" \ + uv run --frozen modal run --env="$MODAL_ENV" \ src/modal/v2_static_runtime_files_smoke.py ) diff --git a/.github/scripts/modal-precompute.sh b/.github/scripts/modal-precompute.sh index 9d8dd4e79..05296cab0 100755 --- a/.github/scripts/modal-precompute.sh +++ b/.github/scripts/modal-precompute.sh @@ -26,7 +26,7 @@ if [ -z "${GITHUB_OUTPUT:-}" ]; then exit 1 fi -RUN_COMMAND=(uv run modal run --env="$MODAL_ENV" src/modal/precompute_app.py) +RUN_COMMAND=(uv run --frozen modal run --env="$MODAL_ENV" src/modal/precompute_app.py) if [[ "$FORCE" == "true" || "$FORCE" == "1" ]]; then RUN_COMMAND+=(--force) fi diff --git a/.github/scripts/modal-record-deployment.sh b/.github/scripts/modal-record-deployment.sh index 87c6f70ba..db0873db5 100755 --- a/.github/scripts/modal-record-deployment.sh +++ b/.github/scripts/modal-record-deployment.sh @@ -16,6 +16,6 @@ if [ -z "${POLICYENGINE_ARTIFACT_BUCKET:-}" ]; then fi echo "Recording deployment marker for ${ENVIRONMENT} (manifest ${MANIFEST_DIGEST})" -uv run python -m src.modal.utils.record_deployment \ +uv run --frozen python -m src.modal.utils.record_deployment \ --environment "$ENVIRONMENT" \ --manifest-digest "$MANIFEST_DIGEST" diff --git a/.github/scripts/modal-setup-environments.sh b/.github/scripts/modal-setup-environments.sh index bb88c4668..6e02930f9 100755 --- a/.github/scripts/modal-setup-environments.sh +++ b/.github/scripts/modal-setup-environments.sh @@ -5,7 +5,7 @@ set -euo pipefail # Create staging environment if it doesn't exist -uv run modal environment create staging 2>/dev/null || echo "staging environment already exists" +uv run --frozen modal environment create staging 2>/dev/null || echo "staging environment already exists" # main environment exists by default echo "Modal environments ready" diff --git a/.github/scripts/modal-sync-secrets.sh b/.github/scripts/modal-sync-secrets.sh index d56ab48ab..8ec3da969 100755 --- a/.github/scripts/modal-sync-secrets.sh +++ b/.github/scripts/modal-sync-secrets.sh @@ -56,7 +56,7 @@ fi # environment in both staging and production. Synchronize that exact resource # during deployment and stop immediately if the update fails; runtime storage # and observability error handling remain independent of this deployment step. -uv run modal secret create gcp-credentials \ +uv run --frozen modal secret create gcp-credentials \ "GOOGLE_APPLICATION_CREDENTIALS_JSON=$GCP_CREDENTIALS_JSON" \ --env="main" \ --force @@ -72,7 +72,7 @@ if [ -n "$NORMALIZED_ISSUER" ]; then esac fi -uv run modal secret create policyengine-gateway-auth \ +uv run --frozen modal secret create policyengine-gateway-auth \ "GATEWAY_AUTH_ISSUER=$NORMALIZED_ISSUER" \ "GATEWAY_AUTH_AUDIENCE=${GATEWAY_AUTH_AUDIENCE:-}" \ "GATEWAY_AUTH_REQUIRED=${GATEWAY_AUTH_REQUIRED:-}" \ diff --git a/.github/scripts/simulation-run-integ-tests.sh b/.github/scripts/simulation-run-integ-tests.sh index c4426b469..26cc52cde 100755 --- a/.github/scripts/simulation-run-integ-tests.sh +++ b/.github/scripts/simulation-run-integ-tests.sh @@ -122,8 +122,8 @@ fi if [ "$ENVIRONMENT" = "beta" ]; then echo "Running all simulation integration tests (including beta_only)" - uv run pytest tests/simulation/ -v + uv run --frozen pytest tests/simulation/ -v else echo "Running simulation integration tests (excluding beta_only)" - uv run pytest tests/simulation/ -v -m "not beta_only" + uv run --frozen pytest tests/simulation/ -v -m "not beta_only" fi diff --git a/.github/scripts/stage12-cache-precompute.sh b/.github/scripts/stage12-cache-precompute.sh index 2b94ad263..01ab897c2 100755 --- a/.github/scripts/stage12-cache-precompute.sh +++ b/.github/scripts/stage12-cache-precompute.sh @@ -11,7 +11,7 @@ modal_environment="${1:?Modal environment is required}" output_file="$(mktemp)" trap 'rm -f "${output_file}"' EXIT -uv run modal run --env="${modal_environment}" src/modal/stage12_precompute_app.py 2>&1 | tee "${output_file}" +uv run --frozen modal run --env="${modal_environment}" src/modal/stage12_precompute_app.py 2>&1 | tee "${output_file}" digest_line="$(tr -d '\r' <"${output_file}" | grep '^STAGE12_CACHE_MANIFEST_DIGEST=' | tail -n 1 || true)" if [[ -z "${digest_line}" ]]; then echo "Stage 12 precompute returned no cache manifest digest" >&2 diff --git a/.github/scripts/stage12-sync-modal-secrets.sh b/.github/scripts/stage12-sync-modal-secrets.sh index 5954fd68a..aefc2a8e8 100755 --- a/.github/scripts/stage12-sync-modal-secrets.sh +++ b/.github/scripts/stage12-sync-modal-secrets.sh @@ -43,11 +43,11 @@ jq -n \ '{GOOGLE_APPLICATION_CREDENTIALS_JSON: $credentials}' \ >"${credentials_modal_file}" -uv run modal secret create stage12-evaluation-runtime \ +uv run --frozen modal secret create stage12-evaluation-runtime \ --from-json "${runtime_modal_file}" \ --env "${modal_environment}" \ --force -uv run modal secret create stage12-evaluation-gcp-credentials \ +uv run --frozen modal secret create stage12-evaluation-gcp-credentials \ --from-json "${credentials_modal_file}" \ --env "${modal_environment}" \ --force diff --git a/.github/scripts/stage12-v2-application-name.sh b/.github/scripts/stage12-v2-application-name.sh index ad637df6e..30983390e 100755 --- a/.github/scripts/stage12-v2-application-name.sh +++ b/.github/scripts/stage12-v2-application-name.sh @@ -6,5 +6,5 @@ set -euo pipefail -name="$(uv run python -c 'from policyengine_simulation_contract.stage12_manifest import v2_application_name; from policyengine_simulation_executor.stage12_bundle import load_stage12_bundle; print(v2_application_name(load_stage12_bundle().bundle.policyengine_version))')" +name="$(uv run --frozen python -c 'from policyengine_simulation_contract.stage12_manifest import v2_application_name; from policyengine_simulation_executor.stage12_bundle import load_stage12_bundle; print(v2_application_name(load_stage12_bundle().bundle.policyengine_version))')" echo "name=${name}" >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/stage12-validate-infrastructure.sh b/.github/scripts/stage12-validate-infrastructure.sh index c64413204..4a13812bf 100755 --- a/.github/scripts/stage12-validate-infrastructure.sh +++ b/.github/scripts/stage12-validate-infrastructure.sh @@ -152,7 +152,7 @@ require_secret_accessor \ "${STAGE12_GCP_CREDENTIALS_SECRET_NAME}" \ "${STAGE12_DEPLOY_SERVICE_ACCOUNT}" -uv run --project projects/policyengine-simulation-entry \ +uv run --frozen --project projects/policyengine-simulation-entry \ python -m policyengine_simulation_entry.stage12_infrastructure \ --database-url-file "${database_url_file}" \ --expected-role "${expected_role}" \ diff --git a/.github/scripts/stage12-validate-v2-workers.sh b/.github/scripts/stage12-validate-v2-workers.sh index df813a8be..3d8cbf209 100755 --- a/.github/scripts/stage12-validate-v2-workers.sh +++ b/.github/scripts/stage12-validate-v2-workers.sh @@ -13,7 +13,7 @@ MODAL_ENV="${1:?Modal environment required}" CACHE_MANIFEST_SHA256="${2:?Stage 12 cache manifest digest required}" output="${RUNNER_TEMP}/stage12-worker-validation.json" -uv run python -m src.modal.utils.validate_v2_worker_release \ +uv run --frozen python -m src.modal.utils.validate_v2_worker_release \ --environment "${MODAL_ENV}" \ --cache-manifest-sha256 "${CACHE_MANIFEST_SHA256}" \ --output "${output}" diff --git a/.github/scripts/update-policyengine-package.sh b/.github/scripts/update-policyengine-package.sh index 97d7a6403..1d187a56e 100755 --- a/.github/scripts/update-policyengine-package.sh +++ b/.github/scripts/update-policyengine-package.sh @@ -301,7 +301,7 @@ done ( cd "$PROJECT_PATH" uv lock --check - uv run --extra test pytest \ + uv run --frozen --extra test pytest \ tests/test_bundle_version_export.py \ tests/test_policyengine_dependency_source.py \ tests/test_modal_bundle_image.py \ diff --git a/.github/workflows/pr-image-smoke.yml b/.github/workflows/pr-image-smoke.yml index b68114579..730b517d5 100644 --- a/.github/workflows/pr-image-smoke.yml +++ b/.github/workflows/pr-image-smoke.yml @@ -58,7 +58,7 @@ jobs: working-directory: projects/policyengine-simulation-executor env: HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} - run: uv run python -m src.modal.utils.validate_hf_access + run: uv run --frozen python -m src.modal.utils.validate_hf_access - name: Run image smokes env: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 52331974f..c68cf38f4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -69,11 +69,11 @@ jobs: - name: Install dependencies working-directory: ${{ matrix.project }} - run: uv sync --extra test + run: uv sync --frozen --extra test - name: Run tests working-directory: ${{ matrix.project }} - run: uv run pytest tests/ -v + run: uv run --frozen pytest tests/ -v lint: name: Lint @@ -97,10 +97,10 @@ jobs: run: uv tool install ruff==0.15.22 - name: Run ruff format check - run: uv run ruff format --check projects/*/src libs/*/src + run: uv run --frozen ruff format --check projects/*/src libs/*/src - name: Run ruff check - run: uv run ruff check . + run: uv run --frozen ruff check . quality-guards: name: Quality guards @@ -228,17 +228,22 @@ jobs: run: | ./scripts/generate-clients.sh + # After the clients exist: policyengine-apis-integ depends on the generated + # client by path, so its lock can only be checked once it is generated. + - name: Check every uv.lock matches its pyproject.toml + run: .github/scripts/check-uv-locks.sh + - name: Start services run: .github/scripts/start-local-services.sh deployment/docker-compose.yml - name: Install integration test dependencies working-directory: projects/policyengine-apis-integ - run: uv sync --extra test + run: uv sync --frozen --extra test - name: Run integration tests (local services only) working-directory: projects/policyengine-apis-integ # Run tests against local Docker services only (not Modal staging) - run: uv run pytest tests/ -v -m "not requires_gcp and not beta_only" + run: uv run --frozen pytest tests/ -v -m "not requires_gcp and not beta_only" - name: Show service logs on failure if: failure() diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 70bc8f111..9de0a3eff 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -128,7 +128,7 @@ jobs: working-directory: projects/policyengine-simulation-executor env: HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} - run: uv run python -m src.modal.utils.validate_hf_access + run: uv run --frozen python -m src.modal.utils.validate_hf_access - name: Synchronize the Modal UK private-data credential if: ${{ inputs.deploy_existing_stack || inputs.deploy_stage12_v2 }} @@ -137,7 +137,7 @@ jobs: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} - run: uv run modal secret create pe-uk-private-hf-read-token "HUGGING_FACE_TOKEN=${PE_UK_PRIVATE_HF_READ_TOKEN}" --env="${{ inputs.modal_environment }}" --force + run: uv run --frozen modal secret create pe-uk-private-hf-read-token "HUGGING_FACE_TOKEN=${PE_UK_PRIVATE_HF_READ_TOKEN}" --env="${{ inputs.modal_environment }}" --force - name: Synchronize Modal runtime secrets if: ${{ inputs.deploy_existing_stack }} @@ -257,7 +257,7 @@ jobs: - name: Run entrypoint unit tests working-directory: ${{ env.PROJECT_DIR }} - run: uv run pytest tests/ -v + run: uv run --frozen pytest tests/ -v deploy_gateway: name: Deploy and unit test Modal gateway @@ -293,11 +293,11 @@ jobs: OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }} OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }} POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }} - run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/policyengine_simulation_gateway/app.py + run: uv run --frozen modal deploy --env="${{ inputs.modal_environment }}" src/policyengine_simulation_gateway/app.py - name: Run gateway unit tests working-directory: projects/policyengine-simulation-gateway - run: uv run pytest tests/ -v + run: uv run --frozen pytest tests/ -v - name: Resolve gateway URL id: url @@ -367,11 +367,11 @@ jobs: OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }} OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }} POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }} - run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/modal/app.py + run: uv run --frozen modal deploy --env="${{ inputs.modal_environment }}" src/modal/app.py - name: Run executor unit tests working-directory: projects/policyengine-simulation-executor - run: uv run pytest tests/ -v + run: uv run --frozen pytest tests/ -v update_routing: name: Publish routing after all services pass @@ -429,7 +429,7 @@ jobs: if [[ "${FORCE_LATEST}" == "true" ]]; then args+=(--force-latest) fi - uv run python -m src.modal.utils.update_version_registry "${args[@]}" \ + uv run --frozen python -m src.modal.utils.update_version_registry "${args[@]}" \ | tee "${RUNNER_TEMP}/routing-plan.out" grep -E '^(before_sha256|after_sha256|advances_latest)=' \ "${RUNNER_TEMP}/routing-plan.out" >> "$GITHUB_OUTPUT" @@ -441,7 +441,7 @@ jobs: MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} MODAL_ENVIRONMENT: ${{ inputs.modal_environment }} run: | - uv run python -m src.modal.utils.update_version_registry \ + uv run --frozen python -m src.modal.utils.update_version_registry \ --environment "${MODAL_ENVIRONMENT}" \ --record-plan "${RUNNER_TEMP}/routing-plan/plan.json" @@ -460,7 +460,7 @@ jobs: MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} MODAL_ENVIRONMENT: ${{ inputs.modal_environment }} run: | - uv run python -m src.modal.utils.update_version_registry \ + uv run --frozen python -m src.modal.utils.update_version_registry \ --environment "${MODAL_ENVIRONMENT}" \ --apply-plan "${RUNNER_TEMP}/routing-plan/plan.json" @@ -491,7 +491,7 @@ jobs: env: HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} GCP_CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS_JSON }} - run: uv run python -m src.modal.utils.release_sentinel expected --output "${RUNNER_TEMP}/release-sentinel.json" + run: uv run --frozen python -m src.modal.utils.release_sentinel expected --output "${RUNNER_TEMP}/release-sentinel.json" - name: Check the sentinel names this deployment's bundle env: @@ -593,7 +593,7 @@ jobs: - name: Validate the packaged PolicyEngine.py bundle working-directory: projects/policyengine-simulation-executor - run: uv run python -m src.modal.utils.extract_stage12_bundle --expect "policyengine_version=${{ needs.prepare.outputs.policyengine_version }}" --expect "countries.us.country_package_version=${{ needs.prepare.outputs.us_version }}" --expect "countries.uk.country_package_version=${{ needs.prepare.outputs.uk_version }}" --output stage12-bundle.json + run: uv run --frozen python -m src.modal.utils.extract_stage12_bundle --expect "policyengine_version=${{ needs.prepare.outputs.policyengine_version }}" --expect "countries.us.country_package_version=${{ needs.prepare.outputs.us_version }}" --expect "countries.uk.country_package_version=${{ needs.prepare.outputs.uk_version }}" --output stage12-bundle.json - name: Select the bundle-derived v2 application name id: application @@ -641,7 +641,7 @@ jobs: OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }} OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }} POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }} - run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/modal/v2_app.py + run: uv run --frozen modal deploy --env="${{ inputs.modal_environment }}" src/modal/v2_app.py - name: Validate installed bundles, secrets, datasets, and calculations id: validation @@ -688,7 +688,7 @@ jobs: env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} - run: uv run python -m src.modal.utils.update_v2_version_manifest --environment "${{ inputs.modal_environment }}" --manifest-name simulation-api-v2-version-manifest --validation-file "${RUNNER_TEMP}/stage12-worker-validation/stage12-worker-validation.json" + run: uv run --frozen python -m src.modal.utils.update_v2_version_manifest --environment "${{ inputs.modal_environment }}" --manifest-name simulation-api-v2-version-manifest --validation-file "${RUNNER_TEMP}/stage12-worker-validation/stage12-worker-validation.json" stage12_v2_ready: name: Require requested Stage 12 v2 deployment to succeed @@ -763,7 +763,7 @@ jobs: SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} run: | uv sync --frozen - uv run python -m src.modal.utils.release_sentinel check \ + uv run --frozen python -m src.modal.utils.release_sentinel check \ --base-url "${{ needs.deploy_entrypoint.outputs.candidate_url }}" \ --expected "${RUNNER_TEMP}/sentinel/release-sentinel.json" @@ -830,7 +830,7 @@ jobs: - name: Run authenticated deployment tests working-directory: projects/policyengine-simulation-entry - run: uv run pytest authenticated_tests/ -v + run: uv run --frozen pytest authenticated_tests/ -v promote_entrypoint: name: Promote and verify Cloud Run entrypoint @@ -921,7 +921,7 @@ jobs: SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} run: | uv sync --frozen - uv run python -m src.modal.utils.release_sentinel check \ + uv run --frozen python -m src.modal.utils.release_sentinel check \ --base-url "${STABLE_URL}" \ --expected "${RUNNER_TEMP}/sentinel/release-sentinel.json" @@ -934,7 +934,7 @@ jobs: SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }} SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} SIMULATION_ENTRYPOINT_TEST_ENFORCE_PERMISSIONS: ${{ vars.SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS }} - run: uv run pytest authenticated_tests/ -v + run: uv run --frozen pytest authenticated_tests/ -v - name: Verify configured custom hostname if: ${{ env.CUSTOM_URL != '' }} @@ -950,7 +950,7 @@ jobs: SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }} SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} SIMULATION_ENTRYPOINT_TEST_ENFORCE_PERMISSIONS: ${{ vars.SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS }} - run: uv run pytest authenticated_tests/ -v + run: uv run --frozen pytest authenticated_tests/ -v - name: Restore previous revision after failed verification if: ${{ failure() && steps.promote.outcome == 'success' }} @@ -1008,7 +1008,7 @@ jobs: run: | # Caller workflows serialize this write with every other routing # write in the simulation-deploy-main concurrency group. - uv run python -m src.modal.utils.update_version_registry \ + uv run --frozen python -m src.modal.utils.update_version_registry \ --environment "${MODAL_ENVIRONMENT}" \ --finalize-publication --publication-run "${PUBLICATION_RUN}" \ --release-environment "${RELEASE_ENVIRONMENT}" \ diff --git a/.github/workflows/simulation-rollback.yml b/.github/workflows/simulation-rollback.yml index 2da9255e9..d04d21040 100644 --- a/.github/workflows/simulation-rollback.yml +++ b/.github/workflows/simulation-rollback.yml @@ -143,7 +143,7 @@ jobs: echo "Deployment run ${DEPLOY_RUN_ID} kept no ${TARGET} routing plan." >&2 exit 1 fi - uv run python -m src.modal.utils.update_version_registry \ + uv run --frozen python -m src.modal.utils.update_version_registry \ --environment "${MODAL_ENVIRONMENT}" \ --restore-plan "${plans[@]}" else @@ -151,7 +151,7 @@ jobs: if [[ -n "${EXPECTED_LATEST}" ]]; then args+=(--expected-latest "${EXPECTED_LATEST}") fi - uv run python -m src.modal.utils.update_version_registry "${args[@]}" + uv run --frozen python -m src.modal.utils.update_version_registry "${args[@]}" fi - name: Resolve the unfinished routing publication @@ -161,9 +161,9 @@ jobs: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} run: | - uv run python -m src.modal.utils.update_version_registry \ + uv run --frozen python -m src.modal.utils.update_version_registry \ --environment "${MODAL_ENVIRONMENT}" --print-pending-plan - uv run python -m src.modal.utils.update_version_registry \ + uv run --frozen python -m src.modal.utils.update_version_registry \ --environment "${MODAL_ENVIRONMENT}" --finalize-publication \ --release-environment "${TARGET}" --gates-passed false @@ -175,7 +175,7 @@ jobs: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} run: | - uv run python -m src.modal.utils.update_version_registry \ + uv run --frozen python -m src.modal.utils.update_version_registry \ --environment "${MODAL_ENVIRONMENT}" --print-identity \ | grep -E '^(policyengine_version|app_name|uk_model_version|uk_dataset_sha256)=' \ | tee -a "$GITHUB_OUTPUT" diff --git a/AGENTS.md b/AGENTS.md index a8ba0614c..498175b75 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -140,6 +140,10 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. - `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. (doc-only, not checked: follow it yourself) - `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. (doc-only, not checked: follow it yourself) +### Dependency locking + +- Checked: `sim.dependencies.install-from-lock`, `sim.dependencies.lock-committed`, `sim.dependencies.scheduled-relock`, `sim.dependencies.test-from-lock` + ### CI workflows and scripts - Checked: `sim.ci.multiline-run-is-script`, `sim.ci.shell-scripts-pass-shellcheck`, `sim.ci.workflow-scripts-tested` diff --git a/Makefile b/Makefile index a886fb14e..94484d22c 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ test: libs/policyengine-stage12-persistence \ libs/policyengine-fastapi; do \ echo "Testing $$proj..."; \ - (cd "$$proj" && uv sync --extra test && uv run pytest tests/ -v) || exit 1; \ + (cd "$$proj" && uv sync --frozen --extra test && uv run --frozen pytest tests/ -v) || exit 1; \ done @echo "✅ All unit tests passed" @@ -100,7 +100,7 @@ test-service: ifndef service @echo "Please specify service: make test-service service=simulation-executor" else - docker-compose -f deployment/docker-compose.yml run --rm $(service) sh -c "cd /app/projects/policyengine-$(service) && uv run --extra test pytest" + docker-compose -f deployment/docker-compose.yml run --rm $(service) sh -c "cd /app/projects/policyengine-$(service) && uv run --frozen --extra test pytest" endif # Integration testing @@ -108,16 +108,16 @@ test-integration: generate-clients @echo "Running integration tests against local services..." @echo "Make sure services are running with 'make up' first!" @cd projects/policyengine-apis-integ && \ - uv sync --extra test && \ - uv run pytest tests/ -v -m "not requires_gcp" + uv sync --frozen --extra test && \ + uv run --frozen pytest tests/ -v -m "not requires_gcp" test-integration-all: generate-clients @echo "Running all integration tests against local services..." @echo "Make sure services are running with 'make up' first!" @echo "Note: Workflow tests will be skipped without GCP credentials" @cd projects/policyengine-apis-integ && \ - uv sync --extra test && \ - uv run pytest tests/ -v + uv sync --frozen --extra test && \ + uv run --frozen pytest tests/ -v test-integration-with-services: @echo "Starting services and running integration tests..." @@ -192,7 +192,7 @@ push-pr-branch: # Integration tests integ-test: - cd projects/policyengine-apis-integ && uv run pytest + cd projects/policyengine-apis-integ && uv run --frozen pytest # Cleanup clean: diff --git a/changelog_entry.yaml b/changelog_entry.yaml index ffa169491..644dfba5c 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,6 +1,7 @@ - bump: patch changes: added: + - Added four registered dependency rules, checked by the locked-installs quality guard with proof fixtures; every project commits a uv.lock that PR CI checks with uv lock --check, every install in workflows, scripts, Makefiles, Dockerfiles, Compose files and Modal images installs the locked versions, every test command runs through uv run --frozen, and the daily update-dependencies workflow relocks every project; every uv sync and uv run now passes --frozen, and the five sites that cannot install from a lock are registered exceptions - Added the rule sim.boundaries.sqlalchemy-and-alembic-only, with proof from the psycopg adapter of 08ebe1ca; code connects to databases only through SQLAlchemy, never by importing a database driver, and runtime code never changes the schema, which changes only through Alembic migrations; a guard parses every file under projects/*/src, libs/*/src and .github/scripts and rejects driver imports, alembic imports outside a migrations directory, create_all and drop_all, create and drop on tables, SQLAlchemy DDL statements and DDL text in string literals, which extends the Stage 12 DDL check's pattern to the gateway and every library - Made the budget's reform-minus-baseline totals log a warning when the baseline and reform household frames carry different weights, since the change is then weighted by the reform's weights alone - Added weighted-microdata rules to the registry, with proof from the #767 household count; the executor's code never aggregates a weight column as a value, reads raw values from weighted frames only in simulation_output_common.py (the UK local-authority breakdown's plain household frame is its one recorded exception there, in a list that may only shrink), and its output-builder tests build entity frames through one shared factory with non-uniform weights, while a doc-only rule records that weights, values, labels and counts each allow different operations diff --git a/docs/engineering/skills/README.md b/docs/engineering/skills/README.md index bc64328ee..f0f10b3bd 100644 --- a/docs/engineering/skills/README.md +++ b/docs/engineering/skills/README.md @@ -11,6 +11,8 @@ Current skills: - `ci-workflows.md`: one line of code per workflow `run:` block, tested workflow scripts, and shellcheck. +- `dependencies.md`: committed `uv.lock` files, installing and testing from + them, the daily relock, and how to register an exception. - `database-access.md`: SQLAlchemy-only database connections, schema changes only through Alembic migrations, and the guard that checks both. - `github-prs.md`: same-repository draft PR workflow, issue linkage, PR head diff --git a/docs/engineering/skills/dependencies.md b/docs/engineering/skills/dependencies.md new file mode 100644 index 000000000..1f75b2880 --- /dev/null +++ b/docs/engineering/skills/dependencies.md @@ -0,0 +1,102 @@ +# Dependency Locking + +Read this before adding or changing anything that installs Python packages or +runs tests: a workflow step, a `.github/scripts/` or `scripts/` script, a +Makefile target, a Dockerfile, a Compose file or a Modal image definition. + +Every project directory (`projects/*`, `libs/*`) is its own uv project with its +own `uv.lock`. The repository root has no project. Four rules keep every +environment on the locked versions. The `locked-installs` quality guard +(`scripts/rules/locked_installs.py`) checks all four on every pull request. + +## We use uv.lock + +`sim.dependencies.lock-committed`: every directory with a `pyproject.toml` +commits a `uv.lock` that matches it. `.github/scripts/check-uv-locks.sh` runs +`uv lock --check --project ` for every tracked `pyproject.toml`, so a new +project is checked without editing the workflow. It runs in the PR +`test-integration` job, after the API clients are generated, because +`policyengine-apis-integ` depends on the generated client by path. + +After editing a `pyproject.toml`, run `uv lock` in that directory and commit +the `uv.lock`. + +## We install from uv.lock + +`sim.dependencies.install-from-lock`: every install installs exactly the +locked versions: + +- `uv sync --frozen` (or `--locked`); +- `uv run --frozen` (or `--locked`), without `--with` or `--no-project`; +- in a Modal image, `uv_sync(..., frozen=True)`. + +The guard reports any other installer: `pip install`, `uv pip`, `uv sync` or +`uv run` without `--frozen`, `uv run --with`, `uvx`, `uv tool`, `pipx`, +`uv add`, `poetry install`, `conda install`, and the Modal methods +`pip_install*`, `uv_pip_install` and `run_commands` that run one of these. +`uv lock` is not an install. Modal's `add_local_python_source` and +`add_local_dir` copy this repository's source files into the image and install +no packages, so they are not reported. + +## We test from uv.lock + +`sim.dependencies.test-from-lock`: every test command (`pytest`, +`python -m pytest`, `coverage run`, `python -m unittest`) runs through +`uv run --frozen`, in the project's locked environment. + +## We relock automatically + +`sim.dependencies.scheduled-relock`: `.github/workflows/update-dependencies.yml` +runs every day. Its `make update` step runs `uv lock --upgrade` in every +`libs/*` and `projects/*` directory with a `pyproject.toml`, and its scripts +open or update the dependency-update pull request. The guard checks this +configuration statically: a `schedule:` workflow whose job, through the +scripts and Make targets it runs, relocks with upgrades, covers every project +directory and runs `gh pr create`. It does not check that the scheduled runs +succeed. + +## What the guard reads + +Every tracked workflow (its `run:` blocks), composite action, shell script, +Makefile, Dockerfile (its `RUN` instructions), Compose file, and Python module +outside tests (for Modal image calls). Before matching, the guard: + +- skips comment lines; +- joins backslash continuations; +- reads each `&&`, `||`, `;`, `|` or backtick segment as its own command; +- removes every single and double quote, so a `match` is written without the + file's quotes (`pip install shellcheck-py==0.11.0.1` for + `pip install "shellcheck-py==0.11.0.1"`); +- skips commands whose first program only prints or edits text: `echo`, + `printf`, `sed`, `grep` and `awk`, after any `NAME=`/`NAME :=` assignment, + `$(` or `$(shell`, and the shell words `if`, `then`, `do`, `else` and `!`. +`TestRepository.test_every_install_capable_file_is_scanned` fails if a file of +one of these kinds is not scanned. Installs that Python code starts through +`subprocess` are not read. + +## Adding an exception + +Only where installing from the lock is not possible. Add a +`[[rule.exception]]` to the rule in `rules/registry.toml`: + +```toml +[[rule.exception]] +path = ".github/workflows/example.yml" # the file +match = "pip install example" # a substring of the one command or call +reason = "Why this site cannot install from uv.lock." +removal = "The condition under which the exception is removed." +# or, instead of removal: expires = 2026-12-31 and issue = +``` + +A tool installed outside any project and pinned to an exact version +(`uv tool install ruff==0.15.22`, `pip install "shellcheck-py==0.11.0.1"`, +`uv tool run ruff@$(RUFF_VERSION)`) may use `category = "pinned-tool"` instead +of a removal condition. The guard checks that `match` contains the pin: +`name==version` or `name@version`, where the version is a release number or a +variable that holds one, never a moving name such as `latest`. + +The guard fails when an exception has no `match`, has neither a removal +condition nor both a review date (`expires`) and a tracking issue (`issue`), or +matches no site (a stale exception). The +current exceptions are listed in `docs/generated/rules.md` under "Exceptions +and review dates". diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 03836cba6..2c6e8bf18 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 45 | 5 | 0 | 15 | 65 | +| mechanical | 49 | 5 | 0 | 15 | 69 | | metric | 0 | 0 | 0 | 0 | 0 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 50 of 65 rules. +**Blocking:** 54 of 69 rules. ## Gaps @@ -73,7 +73,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf ### Exceptions and review dates -- None. +- `sim.dependencies.install-from-lock`: exception for `.github/scripts/update-policyengine-package.sh` (`--with ${PACKAGE}==${LATEST}`) [pinned-tool] (The bundle updater reads the bundle versions of the PolicyEngine release it is about to lock, in an isolated environment outside any project; the release is pinned exactly and is not in any uv.lock yet.) +- `sim.dependencies.install-from-lock`: exception for `.github/workflows/pr.yml` (`pip install shellcheck-py==0.11.0.1`) [pinned-tool] (shellcheck is a binary the ci-scripts guard runs; the quality-guards job has no project environment, so it installs the exact shellcheck-py release.) +- `sim.dependencies.install-from-lock`: exception for `.github/workflows/pr.yml` (`uv tool install ruff==0.15.22`) [pinned-tool] (ruff is a repository-wide tool with no project of its own; ruff.toml sits at the root, which has no pyproject.toml or uv.lock, so the lint job installs the exact version ruff.toml is written for.) +- `sim.dependencies.install-from-lock`: exception for `Makefile` (`uv tool run ruff@$(RUFF_VERSION)`) [pinned-tool] (make format and make check run the ruff release pr.yml pins (RUFF_VERSION is read from its uv tool install ruff==), outside any project, so local results match the lint job.) +- `sim.dependencies.test-from-lock`: exception for `.github/workflows/pr.yml` (`python -m unittest discover -s scripts/rules/tests`); remove when scripts/rules gains a pyproject.toml and uv.lock, or a third-party dependency (The rules tool under scripts/rules uses only the standard library and is not a project: the repository root has no pyproject.toml or uv.lock, so there is no locked environment to run its tests in.) ## Service and package boundaries @@ -373,6 +377,32 @@ flowchart LR e_ruff_per_file_ignore_of_PT_for_scripts_rules_tests___ --> j_CI_pr_yml_lint ``` +## Dependency locking + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.dependencies.install-from-lock` | Every install of a project's Python dependencies, in workflows and the scripts they run, Makefiles, Dockerfiles, Compose files and Modal image definitions, installs exactly the locked versions (uv sync --frozen, uv run --frozen, Modal uv_sync(frozen=True)); any other install is a registered exception for that one site. | mechanical | enforced | yes | quality guard `locked-installs` | `pr.yml:quality-guards` | yes | +| `sim.dependencies.lock-committed` | Every project directory with a pyproject.toml commits a uv.lock that matches it, and PR CI checks every lock with uv lock --check. | mechanical | enforced | yes | quality guard `locked-installs` | `pr.yml:quality-guards`, `pr.yml:test-integration` | yes | +| `sim.dependencies.scheduled-relock` | A scheduled workflow relocks every project's uv.lock with upgrades (uv lock --upgrade) and opens or updates a pull request with the result. | mechanical | enforced | yes | quality guard `locked-installs` | `pr.yml:quality-guards` | yes | +| `sim.dependencies.test-from-lock` | Every test command (pytest, coverage run, python -m unittest) in workflows, scripts, Makefiles, Dockerfiles and Compose files runs through uv run --frozen, in the project's locked environment; any other test run is a registered exception for that one site. | mechanical | enforced | yes | quality guard `locked-installs` | `pr.yml:quality-guards` | yes | + +```mermaid +flowchart LR + r_sim_dependencies_install_from_lock("sim.dependencies.install-from-lock (enforced)") + e_quality_guard_locked_installs["quality guard locked-installs"] + j_CI_pr_yml_quality_guards["CI pr.yml:quality-guards"] + r_sim_dependencies_lock_committed("sim.dependencies.lock-committed (enforced)") + j_CI_pr_yml_test_integration["CI pr.yml:test-integration"] + r_sim_dependencies_scheduled_relock("sim.dependencies.scheduled-relock (enforced)") + r_sim_dependencies_test_from_lock("sim.dependencies.test-from-lock (enforced)") + r_sim_dependencies_install_from_lock --> e_quality_guard_locked_installs + e_quality_guard_locked_installs --> j_CI_pr_yml_quality_guards + r_sim_dependencies_lock_committed --> e_quality_guard_locked_installs + e_quality_guard_locked_installs --> j_CI_pr_yml_test_integration + r_sim_dependencies_scheduled_relock --> e_quality_guard_locked_installs + r_sim_dependencies_test_from_lock --> e_quality_guard_locked_installs +``` + ## CI workflows and scripts | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | diff --git a/projects/policyengine-simulation-entry/tests/test_check_uv_locks_script.py b/projects/policyengine-simulation-entry/tests/test_check_uv_locks_script.py new file mode 100644 index 000000000..b8d8eed6d --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_check_uv_locks_script.py @@ -0,0 +1,51 @@ +"""Tests for .github/scripts/check-uv-locks.sh, which checks every uv.lock.""" + +from __future__ import annotations + +import subprocess +from typing import TYPE_CHECKING + +from workflow_script_support import FakeCommands, run_script + +if TYPE_CHECKING: + from pathlib import Path + +SCRIPT = "check-uv-locks.sh" + + +def _repository(tmp_path: Path) -> Path: + repository = tmp_path / "repository" + for directory in ("libs/one", "projects/two", "scripts/tests/fixtures/three"): + (repository / directory).mkdir(parents=True) + (repository / directory / "pyproject.toml").write_text("[project]\n") + subprocess.run(["git", "init", "-q"], cwd=repository, check=True) + subprocess.run(["git", "add", "."], cwd=repository, check=True) + return repository + + +def test_checks_the_lock_of_every_tracked_project(tmp_path: Path) -> None: + repository = _repository(tmp_path) + fakes = FakeCommands.create(tmp_path) + fakes.add("uv") + + result = run_script(SCRIPT, env=fakes.env(), cwd=repository) + + assert result.returncode == 0, result.stderr + assert fakes.calls() == [ + ["uv", "lock", "--check", "--project", "libs/one"], + ["uv", "lock", "--check", "--project", "projects/two"], + ] + + +def test_fails_after_checking_every_project_when_one_lock_is_stale( + tmp_path: Path, +) -> None: + repository = _repository(tmp_path) + fakes = FakeCommands.create(tmp_path) + fakes.add("uv", 'if [[ "$5" == "libs/one" ]]; then exit 1; fi\n') + + result = run_script(SCRIPT, env=fakes.env(), cwd=repository) + + assert result.returncode == 1 + assert "run uv lock --project libs/one" in result.stderr + assert len(fakes.calls("uv")) == 2 diff --git a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py index e67c64012..9653c8aea 100644 --- a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py +++ b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py @@ -343,7 +343,9 @@ def test_v2_application_name_comes_from_the_packaged_bundle(fakes, tmp_path): assert result.returncode == 0, result.stderr assert _output(output) == "name=policyengine-simulation-v2-py4-1-0\n" - assert fakes.calls() == [["uv", "run", "python", "-c", V2_APPLICATION_CODE]] + assert fakes.calls() == [ + ["uv", "run", "--frozen", "python", "-c", V2_APPLICATION_CODE] + ] def test_v2_application_name_fails_without_writing_when_the_bundle_fails( @@ -383,6 +385,7 @@ def test_validate_v2_workers_records_the_validation_file(fakes, tmp_path): [ "uv", "run", + "--frozen", "python", "-m", "src.modal.utils.validate_v2_worker_release", diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index f6d88ff7a..de151fae2 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -617,7 +617,7 @@ def test_permission_enforcement_reaches_candidate_and_authenticated_tests(): # Every deployed authentication run must know whether the target revision # enforces permissions, so its enforcement check runs or skips correctly. authenticated_runs = reusable_workflow.count( - "run: uv run pytest authenticated_tests/ -v" + "run: uv run --frozen pytest authenticated_tests/ -v" ) assert authenticated_runs == 3 assert ( diff --git a/projects/policyengine-simulation-entry/tests/test_routing_workflow_invariants.py b/projects/policyengine-simulation-entry/tests/test_routing_workflow_invariants.py index 43f47b669..2c1c5718c 100644 --- a/projects/policyengine-simulation-entry/tests/test_routing_workflow_invariants.py +++ b/projects/policyengine-simulation-entry/tests/test_routing_workflow_invariants.py @@ -164,13 +164,17 @@ def _foreground_commands(line: str) -> list[list[str]]: def _assert_supported_prefix(command: list[str], index: int) -> None: prefix = command[:index] if command[index] == REGISTRY_COMMAND: - assert prefix in (["uv", "run", "python", "-m"], ["python", "-m"]), ( - f"Unsupported routing shell prefix: {prefix}" - ) + assert prefix in ( + ["uv", "run", "--frozen", "python", "-m"], + ["uv", "run", "python", "-m"], + ["python", "-m"], + ), f"Unsupported routing shell prefix: {prefix}" else: - assert prefix in (["uv", "run", "python"], ["python"]), ( - f"Unsupported routing shell prefix: {prefix}" - ) + assert prefix in ( + ["uv", "run", "--frozen", "python"], + ["uv", "run", "python"], + ["python"], + ), f"Unsupported routing shell prefix: {prefix}" def _expand_arrays(tokens: list[str], arrays: dict[str, list[str]]) -> tuple[str, ...]: diff --git a/projects/policyengine-simulation-entry/tests/test_simulation_rollback.py b/projects/policyengine-simulation-entry/tests/test_simulation_rollback.py index 8f24c2564..a6ef40dd1 100644 --- a/projects/policyengine-simulation-entry/tests/test_simulation_rollback.py +++ b/projects/policyengine-simulation-entry/tests/test_simulation_rollback.py @@ -244,7 +244,7 @@ def test_rollback_verifies_the_whole_restored_identity(): def _restore_script() -> str: restore = _rollback_steps()[_step_index(_rollback_steps(), "--restore-plan")] script = restore["run"].split("\nelse\n")[0].split("then\n", 1)[1] - return script.replace("uv run python", "printf '%s\\n'") + return script.replace("uv run --frozen python", "printf '%s\\n'") def _inventory_job(*, job=None, step=None, conclusion="success", target="prod"): diff --git a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py index c570903e2..d427800d9 100644 --- a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py +++ b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py @@ -179,7 +179,7 @@ def test_syncs_expected_hf_key_before_image_smokes(self, tmp_path): assert result.returncode == 0, result.stderr calls = uv_calls_log.read_text(encoding="utf-8").splitlines() assert calls[0] == ( - "run modal secret create pe-uk-private-hf-read-token " + "run --frozen modal secret create pe-uk-private-hf-read-token " f"HUGGING_FACE_TOKEN={FAKE_HF_TOKEN} --env=staging --force" ) assert len(calls) == 4 @@ -403,9 +403,9 @@ def test_creates_gateway_secret_with_normalized_issuer(self, tmp_path): assert result.returncode == 0, result.stderr calls = uv_calls_log.read_text() - assert "run modal secret create gcp-credentials" in calls + assert "run --frozen modal secret create gcp-credentials" in calls assert "--env=main --force" in calls - assert "run modal secret create policyengine-gateway-auth" in calls + assert "run --frozen modal secret create policyengine-gateway-auth" in calls assert "GATEWAY_AUTH_ISSUER=https://tenant.auth0.com/" in calls assert ( "GATEWAY_AUTH_AUDIENCE=https://simulation-api-beta.policyengine.org" @@ -450,7 +450,7 @@ def test_gcp_secret_sync_failure_stops_deployment(self, tmp_path): assert result.returncode != 0 calls = uv_calls_log.read_text() - assert "run modal secret create gcp-credentials" in calls + assert "run --frozen modal secret create gcp-credentials" in calls assert "--env=main --force" in calls @@ -556,7 +556,10 @@ def test_runs_precompute_app_and_exports_digest(self, tmp_path): assert result.returncode == 0, result.stderr run_call = next(call for call in calls if "modal run" in call) - assert "run modal run --env=staging src/modal/precompute_app.py" in run_call + assert ( + "run --frozen modal run --env=staging src/modal/precompute_app.py" + in run_call + ) assert "--force" not in run_call assert "manifest_digest=abc123" in github_out @@ -741,7 +744,8 @@ def test_validates_syntax_and_exports_exact_manifest_digest(self, tmp_path): assert result.returncode == 0, result.stderr assert ( - "run modal run --env=staging src/modal/stage12_precompute_app.py" in calls + "run --frozen modal run --env=staging src/modal/stage12_precompute_app.py" + in calls ) assert "--force" not in calls assert emitted == f"manifest_digest={digest}\n" @@ -754,7 +758,7 @@ def test_rejects_an_invalid_digest(self, tmp_path): assert result.returncode != 0 assert calls.strip() == ( - "run modal run --env=staging src/modal/stage12_precompute_app.py" + "run --frozen modal run --env=staging src/modal/stage12_precompute_app.py" ) assert emitted == "" @@ -847,7 +851,7 @@ def test_invokes_the_recorder_module(self, tmp_path): assert result.returncode == 0, result.stderr assert calls == [ - "run python -m src.modal.utils.record_deployment " + "run --frozen python -m src.modal.utils.record_deployment " "--environment beta --manifest-digest digest-abc" ] @@ -1010,7 +1014,7 @@ def test_exports_us_and_uk_model_versions_to_integration_tests(self, tmp_path): assert result.returncode == 0, f"Script failed: {result.stderr}" log = uv_calls_log.read_text() - assert "run pytest tests/simulation/ -v -m not beta_only" in log + assert "run --frozen pytest tests/simulation/ -v -m not beta_only" in log assert "base=https://example.com" in log assert "us=1.690.7" in log assert "uk=2.88.20" in log diff --git a/projects/policyengine-simulation-executor/tests/test_policyengine_package_update_scripts.py b/projects/policyengine-simulation-executor/tests/test_policyengine_package_update_scripts.py index bff4e25c3..811b1cf05 100644 --- a/projects/policyengine-simulation-executor/tests/test_policyengine_package_update_scripts.py +++ b/projects/policyengine-simulation-executor/tests/test_policyengine_package_update_scripts.py @@ -280,7 +280,10 @@ def test_update_policyengine_package_updates_only_wrapper_requirements( assert "uv lock" in uv_calls assert uv_calls.index("run --isolated") < uv_calls.index("uv lock") assert "lock --check" in uv_calls - assert "run --extra test pytest tests/test_bundle_version_export.py" in uv_calls + assert ( + "run --frozen --extra test pytest tests/test_bundle_version_export.py" + in uv_calls + ) assert "checkout -b auto/update-policyengine-4.1.0" in git_log.read_text( encoding="utf-8" ) diff --git a/rules/registry.toml b/rules/registry.toml index 02baddac3..09f0cfd38 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -39,6 +39,10 @@ title = "Weighted microdata" id = "quality" title = "Code quality and tests" +[[category]] +id = "dependencies" +title = "Dependency locking" + [[category]] id = "ci" title = "CI workflows and scripts" @@ -120,7 +124,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [rule.proof] bad_fixture = "rules/fixtures/sim.boundaries.sqlalchemy-and-alembic-only/bad_database_store.py" good_fixture = "rules/fixtures/sim.boundaries.sqlalchemy-and-alembic-only/good_database_store.py" @@ -145,7 +149,7 @@ nodes = ["tests/test_stage12_schema_authority.py::test_simulation_runtimes_conta [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [rule.proof] bad_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py" good_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py" @@ -168,7 +172,7 @@ nodes = ["tests/test_stage12_schema_authority.py::test_simulation_repository_has [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.stage12.no-production-user-associations" @@ -186,7 +190,7 @@ nodes = ["tests/test_stage12_schema_authority.py::test_stage12_runtime_cannot_cr [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.stage12.persistence-typed-dml-only" @@ -204,7 +208,7 @@ nodes = ["tests/test_source_boundaries.py::test_persistence_package_contains_no_ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.stage12.deployment-uses-api-schema" @@ -225,7 +229,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.stage12.deployment-scope" @@ -250,7 +254,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.stage12.production-lock-has-db-runtime" @@ -268,7 +272,7 @@ nodes = ["tests/test_deployment_assets.py::test_production_lock_contains_the_dir [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" # --- Deployment pipeline --------------------------------------------------- @@ -288,7 +292,7 @@ nodes = ["tests/test_deployment_assets.py::test_deployment_scripts_have_valid_sh [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.gcloud-without-terraform" @@ -306,7 +310,7 @@ nodes = ["tests/test_deployment_assets.py::test_deployment_uses_gcloud_workflow_ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.workflow-shell-safety" @@ -327,7 +331,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.revision-tags-valid" @@ -348,7 +352,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.promotion-order-explicit" @@ -369,7 +373,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.smoke-tests-strict" @@ -392,7 +396,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.permission-enforcement-wired" @@ -414,7 +418,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.traffic-changes-exact" @@ -435,7 +439,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.release-identity-smoke" @@ -474,7 +478,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.numerical-sentinel-gate" @@ -495,7 +499,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.routing-publication-journaled" @@ -522,7 +526,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.container-context-minimal" @@ -543,7 +547,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.docker-images-build" @@ -607,7 +611,7 @@ nodes = ["tests/test_openapi_golden.py::test_openapi_spec_matches_golden"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.contracts.regenerate-clients" @@ -635,7 +639,7 @@ kind = "workflow-job" [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test-integration" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.contracts.full-integration-beta-only" @@ -653,7 +657,7 @@ nodes = ["tests/test_deployment_assets.py::test_complete_integration_suite_is_co [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.contracts.clients-publish-deployed-commit" @@ -671,7 +675,7 @@ nodes = ["tests/test_deployment_assets.py::test_client_publication_checks_out_th [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" # --- Observability --------------------------------------------------------- @@ -753,7 +757,7 @@ nodes = ["tests/test_microdata_weight_boundaries.py::test_weights_are_never_aggr [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [rule.proof] bad_fixture = "rules/fixtures/sim.microdata.weights-not-aggregated-as-values/bad_simulation_output_budget.py" good_fixture = "rules/fixtures/sim.microdata.weights-not-aggregated-as-values/good_simulation_output_budget.py" @@ -779,7 +783,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [rule.proof] bad_fixture = "rules/fixtures/sim.microdata.weight-stripping-confined/bad_simulation_output_budget.py" good_fixture = "rules/fixtures/sim.microdata.weight-stripping-confined/good_simulation_output_budget.py" @@ -805,7 +809,7 @@ nodes = ["tests/test_microdata_weight_boundaries.py::test_builder_tests_build_en [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [rule.proof] bad_fixture = "rules/fixtures/sim.microdata.builder-tests-use-weighted-frames/bad_test_simulation_output_budget.py" good_fixture = "rules/fixtures/sim.microdata.builder-tests-use-weighted-frames/good_test_simulation_output_budget.py" @@ -989,7 +993,7 @@ kind = "workflow-job" [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.quality.every-project-tested" @@ -1028,7 +1032,7 @@ nodes = [ [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [rule.proof] bad_fixture = "rules/fixtures/sim.quality.runtime-evaluated-models-resolve/bad_compat_models.py" good_fixture = "rules/fixtures/sim.quality.runtime-evaluated-models-resolve/good_compat_models.py" @@ -1175,6 +1179,138 @@ workflow = ".github/workflows/pr.yml" job = "quality-guards" invocation = "unittest discover -s scripts/rules/tests" +# --- Dependency locking ----------------------------------------------------- + +[[rule]] +id = "sim.dependencies.lock-committed" +category = "dependencies" +statement = "Every project directory with a pyproject.toml commits a uv.lock that matches it, and PR CI checks every lock with uv lock --check." +rationale = "docs/engineering/skills/dependencies.md#we-use-uvlock" +scope = ["projects/*/pyproject.toml", "libs/*/pyproject.toml", "projects/*/uv.lock", "libs/*/uv.lock"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "quality-guard" +name = "locked-installs" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "run_quality_guards.py" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test-integration" +invocation = "check-uv-locks.sh" +[rule.proof] +bad_fixture = "scripts/rules/tests/fixtures/locked_installs/lock_bad" +good_fixture = "scripts/rules/tests/fixtures/locked_installs/lock_good" +test = "scripts/rules/tests/test_locked_installs.py::TestLockedInstallsProof::test_rule_proof_lock_committed" +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "unittest discover -s scripts/rules/tests" + +[[rule]] +id = "sim.dependencies.install-from-lock" +category = "dependencies" +statement = "Every install of a project's Python dependencies, in workflows and the scripts they run, Makefiles, Dockerfiles, Compose files and Modal image definitions, installs exactly the locked versions (uv sync --frozen, uv run --frozen, Modal uv_sync(frozen=True)); any other install is a registered exception for that one site." +rationale = "docs/engineering/skills/dependencies.md#we-install-from-uvlock" +scope = [".github/workflows/*.yml", ".github/scripts/*.sh", "scripts/*.sh", "Makefile", "projects/*/Dockerfile", "projects/*/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "quality-guard" +name = "locked-installs" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "run_quality_guards.py" +[rule.proof] +bad_fixture = "scripts/rules/tests/fixtures/locked_installs/install_bad" +good_fixture = "scripts/rules/tests/fixtures/locked_installs/install_good" +test = "scripts/rules/tests/test_locked_installs.py::TestLockedInstallsProof::test_rule_proof_install_from_lock" +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "unittest discover -s scripts/rules/tests" + +[[rule.exception]] +path = ".github/workflows/pr.yml" +match = "uv tool install ruff==0.15.22" +category = "pinned-tool" +reason = "ruff is a repository-wide tool with no project of its own; ruff.toml sits at the root, which has no pyproject.toml or uv.lock, so the lint job installs the exact version ruff.toml is written for." + +[[rule.exception]] +path = ".github/workflows/pr.yml" +match = "pip install shellcheck-py==0.11.0.1" +category = "pinned-tool" +reason = "shellcheck is a binary the ci-scripts guard runs; the quality-guards job has no project environment, so it installs the exact shellcheck-py release." + +[[rule.exception]] +path = "Makefile" +match = "uv tool run ruff@$(RUFF_VERSION)" +category = "pinned-tool" +reason = "make format and make check run the ruff release pr.yml pins (RUFF_VERSION is read from its uv tool install ruff==), outside any project, so local results match the lint job." + +[[rule.exception]] +path = ".github/scripts/update-policyengine-package.sh" +match = "--with ${PACKAGE}==${LATEST}" +category = "pinned-tool" +reason = "The bundle updater reads the bundle versions of the PolicyEngine release it is about to lock, in an isolated environment outside any project; the release is pinned exactly and is not in any uv.lock yet." + +[[rule]] +id = "sim.dependencies.test-from-lock" +category = "dependencies" +statement = "Every test command (pytest, coverage run, python -m unittest) in workflows, scripts, Makefiles, Dockerfiles and Compose files runs through uv run --frozen, in the project's locked environment; any other test run is a registered exception for that one site." +rationale = "docs/engineering/skills/dependencies.md#we-test-from-uvlock" +scope = [".github/workflows/*.yml", ".github/scripts/*.sh", "scripts/*.sh", "Makefile"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "quality-guard" +name = "locked-installs" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "run_quality_guards.py" +[rule.proof] +bad_fixture = "scripts/rules/tests/fixtures/locked_installs/test_bad" +good_fixture = "scripts/rules/tests/fixtures/locked_installs/test_good" +test = "scripts/rules/tests/test_locked_installs.py::TestLockedInstallsProof::test_rule_proof_test_from_lock" +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "unittest discover -s scripts/rules/tests" + +[[rule.exception]] +path = ".github/workflows/pr.yml" +match = "python -m unittest discover -s scripts/rules/tests" +reason = "The rules tool under scripts/rules uses only the standard library and is not a project: the repository root has no pyproject.toml or uv.lock, so there is no locked environment to run its tests in." +removal = "scripts/rules gains a pyproject.toml and uv.lock, or a third-party dependency" + +[[rule]] +id = "sim.dependencies.scheduled-relock" +category = "dependencies" +statement = "A scheduled workflow relocks every project's uv.lock with upgrades (uv lock --upgrade) and opens or updates a pull request with the result." +rationale = "docs/engineering/skills/dependencies.md#we-relock-automatically" +scope = [".github/workflows/update-dependencies.yml", "Makefile"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "quality-guard" +name = "locked-installs" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "run_quality_guards.py" +[rule.proof] +bad_fixture = "scripts/rules/tests/fixtures/locked_installs/relock_bad" +good_fixture = "scripts/rules/tests/fixtures/locked_installs/relock_good" +test = "scripts/rules/tests/test_locked_installs.py::TestLockedInstallsProof::test_rule_proof_scheduled_relock" +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "unittest discover -s scripts/rules/tests" + # --- CI workflows and scripts ---------------------------------------------- [[rule]] @@ -1287,7 +1423,7 @@ nodes = ["tests/test_ci_complete_gate.py::test_pr_workflow_is_gated", "tests/tes [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" -invocation = "uv run pytest tests/" +invocation = "uv run --frozen pytest tests/" [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "ci-complete" diff --git a/scripts/generate-clients.sh b/scripts/generate-clients.sh index 3130a6364..9ea209387 100755 --- a/scripts/generate-clients.sh +++ b/scripts/generate-clients.sh @@ -17,13 +17,13 @@ generate_client() { cd "${PROJECT_DIR}" # Install build dependencies if not already installed - uv sync --extra build + uv sync --frozen --extra build # The permanent Cloud Run Simulation Entrypoint owns the public schema. if [ "$SERVICE" = "simulation" ]; then - uv run python -m policyengine_simulation_entry.generate_openapi + uv run --frozen python -m policyengine_simulation_entry.generate_openapi else - uv run python -m policyengine_api_${SERVICE//-/_}.generate_openapi + uv run --frozen python -m policyengine_api_${SERVICE//-/_}.generate_openapi fi # Check if OpenAPI spec was created @@ -38,13 +38,13 @@ generate_client() { # Use config file if it exists (for package name override) if [ -f "openapi-python-client.yaml" ]; then - uv run openapi-python-client generate \ + uv run --frozen openapi-python-client generate \ --path artifacts/openapi.json \ --output-path artifacts/clients/python \ --config openapi-python-client.yaml \ --overwrite else - uv run openapi-python-client generate \ + uv run --frozen openapi-python-client generate \ --path artifacts/openapi.json \ --output-path artifacts/clients/python \ --overwrite diff --git a/scripts/rules/load.py b/scripts/rules/load.py index 0a235a2b7..83a5da5ad 100644 --- a/scripts/rules/load.py +++ b/scripts/rules/load.py @@ -296,6 +296,11 @@ def _baseline( return None +def _optional_text(entry: dict[str, Any], key: str) -> str: + value = entry.get(key, "") + return value if isinstance(value, str) else "" + + def _exceptions( c: _Collector, rule_id: str | None, raw: dict[str, Any], label: str ) -> tuple[RuleException, ...]: @@ -311,14 +316,33 @@ def _exceptions( ) continue c.unknown_keys( - rule_id, entry, {"path", "reason", "expires"}, f"{label}.exception" + rule_id, + entry, + {"path", "reason", "expires", "match", "category", "removal", "issue"}, + f"{label}.exception", + ) + expires = ( + c.day(rule_id, entry["expires"], f"{label}.exception.expires") + if "expires" in entry + else None ) - expires = c.day(rule_id, entry.get("expires"), f"{label}.exception.expires") + issue = entry.get("issue") + if issue is not None and (not isinstance(issue, int) or issue <= 0): + c.add( + rule_id, + f"{label}.exception.issue must be a positive issue number", + "write issue = ", + ) + issue = None exceptions.append( RuleException( c.string(rule_id, entry, "path", f"{label}.exception"), c.string(rule_id, entry, "reason", f"{label}.exception"), expires or date.max, + match=_optional_text(entry, "match"), + category=_optional_text(entry, "category"), + removal=_optional_text(entry, "removal"), + issue=issue, ) ) return tuple(exceptions) diff --git a/scripts/rules/locked_installs.py b/scripts/rules/locked_installs.py new file mode 100644 index 000000000..08f7358cf --- /dev/null +++ b/scripts/rules/locked_installs.py @@ -0,0 +1,570 @@ +"""Check that Python packages are locked, installed and tested from uv.lock. + +The guard reads every tracked file that can install Python packages or run +tests: workflows, composite actions, shell scripts, Makefiles, Dockerfiles, +Compose files and Python modules that define Modal images. It checks four rules +(docs/engineering/skills/dependencies.md): + +- every project directory commits a uv.lock, and PR CI runs ``uv lock --check``; +- every install comes from uv.lock (``uv sync --frozen``, ``uv run --frozen``, + Modal ``uv_sync(frozen=True)``), unless a ``[[rule.exception]]`` names the site; +- every test command runs through ``uv run --frozen``; +- a scheduled workflow relocks every uv.lock with upgrades and opens a PR. +""" + +from __future__ import annotations + +import fnmatch +import re +import subprocess +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +from scripts.rules.load import load_registry +from scripts.rules.model import Violation +from scripts.rules.workflows import parse_jobs + +if TYPE_CHECKING: + from collections.abc import Iterable + + from scripts.rules.model import Registry, RuleException + +REPO_ROOT = Path(__file__).resolve().parents[2] + +LOCK_RULE = "sim.dependencies.lock-committed" +INSTALL_RULE = "sim.dependencies.install-from-lock" +TEST_RULE = "sim.dependencies.test-from-lock" +RELOCK_RULE = "sim.dependencies.scheduled-relock" +PINNED_TOOL = "pinned-tool" +PR_WORKFLOW = ".github/workflows/pr.yml" + +_LOCKED_FLAG = re.compile(r"(? str: + """Format as ``path:line: text``.""" + return f"{self.path}:{self.line}: {self.text}" + + +@dataclass +class Findings: + """What the scan found, before exceptions are applied.""" + + scanned: set[str] = field(default_factory=set) + unlocked_installs: list[Site] = field(default_factory=list) + unlocked_tests: list[Site] = field(default_factory=list) + + +def tracked_files(root: Path) -> list[str]: + """Tracked files under ``root``, or every file when it is not a git checkout.""" + if (root / ".git").exists(): + listed = subprocess.run( + ["git", "ls-files"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout + return [path for path in listed.splitlines() if (root / path).is_file()] + return sorted( + str(path.relative_to(root)) for path in root.rglob("*") if path.is_file() + ) + + +def _is_fixture_or_test(path: str) -> bool: + parts = PurePosixPath(path).parts + return "fixtures" in parts or "tests" in parts or "authenticated_tests" in parts + + +def file_kind(path: str) -> str | None: + """Return which kind of install-capable file ``path`` is, or None.""" + pure = PurePosixPath(path) + name = pure.name.lower() + if "fixtures" in pure.parts: + return None + kinds = ( + ( + "workflow", + path.startswith(".github/workflows/") and name.endswith((".yml", ".yaml")), + ), + ( + "action", + path.startswith(".github/actions/") + and name in {"action.yml", "action.yaml"}, + ), + ("shell", name.endswith(".sh")), + ("makefile", name == "makefile" or name.endswith(".mk")), + ("dockerfile", "dockerfile" in name and not name.endswith(".dockerignore")), + ("compose", fnmatch.fnmatch(name, "*compose*.y*ml")), + ("python", name.endswith(".py") and not _is_fixture_or_test(path)), + ) + return next((kind for kind, matches in kinds if matches), None) + + +def _logical_lines(text: str) -> list[tuple[int, str]]: + """Join backslash continuations and drop comment-only lines.""" + joined: list[tuple[int, str]] = [] + pending = "" + start = 0 + for number, raw in enumerate(text.splitlines(), start=1): + stripped = raw.strip() + if not pending and stripped.startswith("#"): + continue + if not pending: + start = number + if stripped.endswith("\\"): + pending += stripped[:-1] + " " + continue + joined.append((start, pending + stripped)) + pending = "" + if pending: + joined.append((start, pending)) + return joined + + +def _commands(text: str) -> list[tuple[int, str]]: + """Split each logical line into the simple commands it runs.""" + commands = [] + for number, logical in _logical_lines(text): + for part in _SEPARATORS.split(logical): + command = part.replace('"', "").replace("'", "").strip() + if command and not _TEXT_TOOL.match(command): + commands.append((number, command)) + return commands + + +def _workflow_commands(text: str) -> list[tuple[int, str]]: + """Return each workflow run-block command with the file line it is on. + + Run blocks are found in file order: each block starts on the first line, + after the previous block's start, that holds the block's first line. + """ + jobs, _problems = parse_jobs(text) + lines = text.splitlines() + commands = [] + cursor = 0 + for job in jobs.values(): + for block in job.run_blocks: + block_lines = block.text.splitlines() + first = next((line.strip() for line in block_lines if line.strip()), "") + start = next( + ( + index + for index in range(cursor, len(lines)) + if first and first in lines[index] + ), + cursor, + ) + cursor = start + 1 + offset = next( + (index for index, line in enumerate(block_lines) if line.strip()), 0 + ) + commands.extend( + (start + 1 + number - 1 - offset, command) + for number, command in _commands(block.text) + ) + return commands + + +def _shell_text(kind: str, text: str) -> list[tuple[int, str]]: + """Return the shell code a file runs, with the line it starts on.""" + if kind == "workflow": + return _workflow_commands(text) + if kind == "dockerfile": + return [ + (number, command) + for number, logical in _logical_lines(text) + if re.match(r"(?i)^RUN\s", logical) + for _, command in _commands(logical[4:]) + ] + return _commands(text) + + +def _unlocked_install(command: str) -> bool: + if any(pattern.search(command) for _, pattern in _UNLOCKED_INSTALLS): + return True + if _UV_SYNC.search(command): + return not _LOCKED_FLAG.search(command) + if _UV_RUN.search(command): + return not _LOCKED_FLAG.search(command) or bool( + _UNLOCKED_RUN_FLAG.search(command) + ) + return False + + +def _unlocked_test(command: str) -> bool: + if not _TEST_COMMAND.search(command): + return False + return not (_UV_RUN.search(command) and _LOCKED_FLAG.search(command)) + + +def _calls(text: str) -> list[tuple[int, str, str]]: + """Return each Modal image call as (line, method, call text).""" + calls = [] + for match in _MODAL_CALL.finditer(text): + depth = 0 + end = match.end() - 1 + for index in range(match.end() - 1, len(text)): + if text[index] == "(": + depth += 1 + elif text[index] == ")": + depth -= 1 + if depth == 0: + end = index + 1 + break + number = text.count("\n", 0, match.start()) + 1 + call_text = text[match.start() + 1 : end].replace('"', "").replace("'", "") + call = " ".join(call_text.split()) + calls.append((number, match.group(1), call)) + return calls + + +def _scan_python(path: str, text: str, findings: Findings) -> None: + for number, method, call in _calls(text): + unlocked = ( + method in _MODAL_INSTALLS + or (method == "uv_sync" and not _MODAL_FROZEN.search(call)) + or (method == "run_commands" and _unlocked_install(call)) + ) + if unlocked: + findings.unlocked_installs.append(Site(path, number, call)) + if method == "run_commands" and _unlocked_test(call): + findings.unlocked_tests.append(Site(path, number, call)) + + +def scan(root: Path, files: Iterable[str] | None = None) -> Findings: + """Scan every install-capable file under ``root``.""" + findings = Findings() + for path in sorted(tracked_files(root) if files is None else files): + kind = file_kind(path) + if kind is None: + continue + findings.scanned.add(path) + text = (root / path).read_text(encoding="utf-8", errors="replace") + if kind == "python": + _scan_python(path, text, findings) + continue + for number, command in _shell_text(kind, text): + if _unlocked_install(command): + findings.unlocked_installs.append(Site(path, number, command)) + if _unlocked_test(command): + findings.unlocked_tests.append(Site(path, number, command)) + return findings + + +def _covers(exception: RuleException, site: Site) -> bool: + return ( + fnmatch.fnmatch(site.path, exception.path) + and bool(exception.match) + and (exception.match in site.text) + ) + + +def _exception_problems(rule_id: str, exception: RuleException) -> list[Violation]: + """Return why an exception does not name a site, a reason and an end.""" + problems = [] + where = f"exception for {exception.path}" + if not exception.match: + problems.append( + Violation( + rule_id, + "locked-installs", + f"{where} names no site", + "add match = a substring of the one command or call it allows", + ) + ) + if exception.category == PINNED_TOOL: + if not _PINNED.search(exception.match): + problems.append( + Violation( + rule_id, + "locked-installs", + f"pinned-tool {where} does not pin an exact version", + "pin the tool with == and include the pin in match", + ) + ) + elif exception.category: + problems.append( + Violation( + rule_id, + "locked-installs", + f"{where} has unknown category '{exception.category}'", + f"use category = '{PINNED_TOOL}' or leave it out", + ) + ) + elif not exception.removal and ( + exception.issue is None or exception.expires == date.max + ): + problems.append( + Violation( + rule_id, + "locked-installs", + f"{where} has no removal condition and no review date with a " + "tracking issue", + "add removal = the condition for removing it, or both expires " + "and issue", + ) + ) + return problems + + +def _apply_exceptions( + registry: Registry, rule_id: str, sites: list[Site], what: str, fix: str +) -> list[Violation]: + rule = next((rule for rule in registry.rules if rule.id == rule_id), None) + exceptions = rule.exceptions if rule is not None else () + violations = [ + problem + for exception in exceptions + for problem in _exception_problems(rule_id, exception) + ] + violations.extend( + Violation(rule_id, "locked-installs", f"{what}: {site}", fix) + for site in sites + if not any(_covers(exception, site) for exception in exceptions) + ) + violations.extend( + Violation( + rule_id, + "locked-installs", + f"exception for {exception.path} ({exception.match}) matches no site", + "remove the stale exception", + ) + for exception in exceptions + if exception.match and not any(_covers(exception, site) for site in sites) + ) + return violations + + +def _makefile_recipes(root: Path, makefile: str) -> dict[str, list[str]]: + recipes: dict[str, list[str]] = {} + path = root / makefile + if not path.is_file(): + return recipes + current = None + for raw in path.read_text(encoding="utf-8").splitlines(): + header = _MAKE_HEADER.match(raw) + if header is not None and not header.group(1).startswith("."): + current = header.group(1) + recipes[current] = [] + elif raw.startswith("\t") and current is not None: + recipes[current].append(raw) + elif raw.strip() and not raw.startswith("\t"): + current = None + return recipes + + +def executed_text(root: Path, text: str, seen: set[str] | None = None) -> str: + """Return ``text`` plus the scripts and root Make targets it runs.""" + seen = set() if seen is None else seen + recipes = _makefile_recipes(root, "Makefile") + pieces = [text] + for script in _SCRIPT_REFERENCE.findall(text): + if script not in seen and (root / script).is_file(): + seen.add(script) + script_text = (root / script).read_text(encoding="utf-8") + pieces.append(executed_text(root, script_text, seen)) + for target in _MAKE_TARGET.findall(text): + if f"make {target}" not in seen and target in recipes: + seen.add(f"make {target}") + pieces.append(executed_text(root, "\n".join(recipes[target]), seen)) + return "\n".join(pieces) + + +def _project_directories(files: list[str]) -> list[str]: + return sorted( + str(PurePosixPath(path).parent) + for path in files + if PurePosixPath(path).name == "pyproject.toml" + and not _is_fixture_or_test(path) + ) + + +def _workflow_jobs(root: Path, files: list[str]) -> list[tuple[str, str, str]]: + """Return (workflow, job id, executed text) for every workflow job.""" + jobs = [] + for path in files: + if file_kind(path) != "workflow": + continue + workflow_text = (root / path).read_text(encoding="utf-8") + parsed, _problems = parse_jobs(workflow_text) + for job_id, job in parsed.items(): + run_text = "\n".join(block.text for block in job.run_blocks) + uses = "\n".join(job.step_uses) + jobs.append((path, job_id, executed_text(root, f"{run_text}\n{uses}"))) + return jobs + + +def lock_violations(root: Path, files: list[str]) -> list[Violation]: + """Rule 1: each project commits a uv.lock and PR CI checks it.""" + tracked = set(files) + violations = [ + Violation( + LOCK_RULE, + "locked-installs", + f"{directory}/pyproject.toml has no committed uv.lock", + f"run uv lock --project {directory} and commit uv.lock", + ) + for directory in _project_directories(files) + if f"{directory}/uv.lock".removeprefix("./") not in tracked + ] + checked = any( + workflow == PR_WORKFLOW and _LOCK_CHECK.search(text) + for workflow, _job, text in _workflow_jobs(root, files) + ) + if not checked: + violations.append( + Violation( + LOCK_RULE, + "locked-installs", + f"no {PR_WORKFLOW} job runs uv lock --check", + "run .github/scripts/check-uv-locks.sh in a PR CI job", + ) + ) + return violations + + +def relock_violations(root: Path, files: list[str]) -> list[Violation]: + """Rule 4: a scheduled workflow relocks every uv.lock and opens a PR.""" + projects = _project_directories(files) + for path in files: + if file_kind(path) != "workflow": + continue + if not re.search( + r"^\s+schedule:", (root / path).read_text(encoding="utf-8"), re.MULTILINE + ): + continue + for workflow, _job, text in _workflow_jobs(root, [path]): + if not (_RELOCK.search(text) and _OPENS_PR.search(text)): + continue + globs = re.findall(r"[\w./*-]*pyproject\.toml", text) + uncovered = [ + directory + for directory in projects + if directory != "." + and not any( + fnmatch.fnmatch(f"{directory}/pyproject.toml", glob) + for glob in globs + ) + ] + if not uncovered: + return [] + return [ + Violation( + RELOCK_RULE, + "locked-installs", + f"{workflow} does not relock {directory}/uv.lock", + "relock every project directory in the scheduled job", + ) + for directory in uncovered + ] + return [ + Violation( + RELOCK_RULE, + "locked-installs", + "no scheduled workflow relocks uv.lock with upgrades and opens a PR", + "add a schedule: workflow that runs uv lock --upgrade and gh pr create", + ) + ] + + +def find_violations(root: Path, registry: Registry) -> list[Violation]: + """Run all four checks against the files under ``root``.""" + files = tracked_files(root) + findings = scan(root, files) + return [ + *lock_violations(root, files), + *_apply_exceptions( + registry, + INSTALL_RULE, + findings.unlocked_installs, + "unlocked install", + "install from uv.lock (uv sync --frozen, uv run --frozen, " + "uv_sync(frozen=True)) or register a [[rule.exception]] for the site", + ), + *_apply_exceptions( + registry, + TEST_RULE, + findings.unlocked_tests, + "test command outside uv.lock", + "run it through uv run --frozen or register a [[rule.exception]]", + ), + *relock_violations(root, files), + ] + + +def run_check(root: Path = REPO_ROOT) -> list[Violation]: + """Load the registry under ``root`` and return every violation.""" + registry, violations = load_registry(root) + if registry is None: + return violations + return find_violations(root, registry) + + +def check() -> list[str]: + """Adapter for scripts/run_quality_guards.py.""" + return [str(violation) for violation in run_check(REPO_ROOT)] diff --git a/scripts/rules/model.py b/scripts/rules/model.py index 6f3bc1397..736e72a39 100644 --- a/scripts/rules/model.py +++ b/scripts/rules/model.py @@ -211,11 +211,22 @@ class Baseline: @dataclass(frozen=True) class RuleException: - """A temporary carve-out of ``path`` from a rule; the guard fails once expired.""" + """A carve-out of ``path`` from a rule; the guard fails once expired. + + ``match`` narrows the carve-out to one site in the file (a substring of the + command or call), ``category`` names a class of allowed site (for example + ``pinned-tool``), ``removal`` says when the exception can go, and ``issue`` + is the tracking issue for a dated exception. Guards that need these fields + check them; the registry loader only reads them. + """ path: str reason: str expires: date + match: str = "" + category: str = "" + removal: str = "" + issue: int | None = None @dataclass(frozen=True) diff --git a/scripts/rules/render.py b/scripts/rules/render.py index e22985d5f..df3cb42e4 100644 --- a/scripts/rules/render.py +++ b/scripts/rules/render.py @@ -7,9 +7,10 @@ from __future__ import annotations import re +from datetime import date from pathlib import Path -from scripts.rules.model import Kind, Registry, Rule, Status, Tier +from scripts.rules.model import Kind, Registry, Rule, RuleException, Status, Tier RULES_MD = Path("docs/generated/rules.md") AGENTS_MD = Path("AGENTS.md") @@ -25,6 +26,24 @@ def _cell(text: str) -> str: return text.replace("|", "\\|").replace("\n", " ") +def _describe_exception(exception: RuleException) -> str: + """Say which site an exception covers, why, and when it ends.""" + site = f"`{exception.path}`" + if exception.match: + site += f" (`{exception.match}`)" + if exception.category: + site += f" [{exception.category}]" + if exception.expires != date.max: + site += f" until {exception.expires}" + ends = [] + if exception.issue is not None: + ends.append(f"tracked in #{exception.issue}") + if exception.removal: + ends.append(f"remove when {exception.removal}") + ending = f"; {'; '.join(ends)}" if ends else "" + return f"exception for {site}{ending} ({exception.reason})" + + def _ordered(registry: Registry) -> list[tuple[str, str, list[Rule]]]: """Categories in declared order, each with its rules sorted by id.""" groups = [] @@ -193,9 +212,7 @@ def section(title: str, items: list[str]) -> None: ], ) dated = [f"- `{r.id}`: review by {r.review_by}" for r in rules if r.review_by] + [ - f"- `{r.id}`: exception for `{e.path}` until {e.expires} ({e.reason})" - for r in rules - for e in r.exceptions + f"- `{r.id}`: {_describe_exception(e)}" for r in rules for e in r.exceptions ] section("Exceptions and review dates", sorted(dated)) diff --git a/scripts/rules/tests/fixtures/locked_installs/install_bad/.github/workflows/ci.yml b/scripts/rules/tests/fixtures/locked_installs/install_bad/.github/workflows/ci.yml new file mode 100644 index 000000000..586ffc7d0 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/install_bad/.github/workflows/ci.yml @@ -0,0 +1,8 @@ +name: CI +on: pull_request +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: uv sync --extra test + - run: pip install requests diff --git a/scripts/rules/tests/fixtures/locked_installs/install_bad/Dockerfile b/scripts/rules/tests/fixtures/locked_installs/install_bad/Dockerfile new file mode 100644 index 000000000..c40447b97 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/install_bad/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.13 +RUN pip install policyengine diff --git a/scripts/rules/tests/fixtures/locked_installs/install_bad/app/modal_app.py b/scripts/rules/tests/fixtures/locked_installs/install_bad/app/modal_app.py new file mode 100644 index 000000000..d75099dc2 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/install_bad/app/modal_app.py @@ -0,0 +1,3 @@ +import modal + +image = modal.Image.debian_slim().pip_install("policyengine-us") diff --git a/scripts/rules/tests/fixtures/locked_installs/install_good/.github/workflows/ci.yml b/scripts/rules/tests/fixtures/locked_installs/install_good/.github/workflows/ci.yml new file mode 100644 index 000000000..d8b595310 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/install_good/.github/workflows/ci.yml @@ -0,0 +1,8 @@ +name: CI +on: pull_request +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: uv sync --frozen --extra test + - run: uv run --frozen pytest tests/ diff --git a/scripts/rules/tests/fixtures/locked_installs/install_good/Dockerfile b/scripts/rules/tests/fixtures/locked_installs/install_good/Dockerfile new file mode 100644 index 000000000..8ad9fd182 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/install_good/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.13 +RUN uv sync --frozen --no-dev diff --git a/scripts/rules/tests/fixtures/locked_installs/install_good/app/modal_app.py b/scripts/rules/tests/fixtures/locked_installs/install_good/app/modal_app.py new file mode 100644 index 000000000..e31b624a2 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/install_good/app/modal_app.py @@ -0,0 +1,3 @@ +import modal + +image = modal.Image.debian_slim().uv_sync(uv_project_dir=".", frozen=True) diff --git a/scripts/rules/tests/fixtures/locked_installs/lock_bad/.github/workflows/pr.yml b/scripts/rules/tests/fixtures/locked_installs/lock_bad/.github/workflows/pr.yml new file mode 100644 index 000000000..465a3795b --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/lock_bad/.github/workflows/pr.yml @@ -0,0 +1,7 @@ +name: PR +on: pull_request +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: uv sync --frozen diff --git a/scripts/rules/tests/fixtures/locked_installs/lock_bad/app/pyproject.toml b/scripts/rules/tests/fixtures/locked_installs/lock_bad/app/pyproject.toml new file mode 100644 index 000000000..756d792fa --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/lock_bad/app/pyproject.toml @@ -0,0 +1,3 @@ +[project] +name = "app" +version = "0" diff --git a/scripts/rules/tests/fixtures/locked_installs/lock_good/.github/workflows/pr.yml b/scripts/rules/tests/fixtures/locked_installs/lock_good/.github/workflows/pr.yml new file mode 100644 index 000000000..f0b7091ed --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/lock_good/.github/workflows/pr.yml @@ -0,0 +1,7 @@ +name: PR +on: pull_request +jobs: + locks: + runs-on: ubuntu-latest + steps: + - run: .github/scripts/check-uv-locks.sh diff --git a/scripts/rules/tests/fixtures/locked_installs/lock_good/app/pyproject.toml b/scripts/rules/tests/fixtures/locked_installs/lock_good/app/pyproject.toml new file mode 100644 index 000000000..756d792fa --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/lock_good/app/pyproject.toml @@ -0,0 +1,3 @@ +[project] +name = "app" +version = "0" diff --git a/scripts/rules/tests/fixtures/locked_installs/lock_good/app/uv.lock b/scripts/rules/tests/fixtures/locked_installs/lock_good/app/uv.lock new file mode 100644 index 000000000..d9914dfa6 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/lock_good/app/uv.lock @@ -0,0 +1 @@ +version = 1 diff --git a/scripts/rules/tests/fixtures/locked_installs/relock_bad/.github/workflows/update.yml b/scripts/rules/tests/fixtures/locked_installs/relock_bad/.github/workflows/update.yml new file mode 100644 index 000000000..a1697521d --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/relock_bad/.github/workflows/update.yml @@ -0,0 +1,10 @@ +name: Update +on: + schedule: + - cron: "0 15 * * *" +jobs: + update: + runs-on: ubuntu-latest + steps: + - run: make update + - run: gh pr create --fill diff --git a/scripts/rules/tests/fixtures/locked_installs/relock_bad/Makefile b/scripts/rules/tests/fixtures/locked_installs/relock_bad/Makefile new file mode 100644 index 000000000..7e42f17d4 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/relock_bad/Makefile @@ -0,0 +1,4 @@ +update: + @for pyproject in libs/*/pyproject.toml; do \ + (cd "$$(dirname $$pyproject)" && uv lock --upgrade); \ + done diff --git a/scripts/rules/tests/fixtures/locked_installs/relock_bad/libs/one/pyproject.toml b/scripts/rules/tests/fixtures/locked_installs/relock_bad/libs/one/pyproject.toml new file mode 100644 index 000000000..52b7b2adf --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/relock_bad/libs/one/pyproject.toml @@ -0,0 +1,3 @@ +[project] +name = "one" +version = "0" diff --git a/scripts/rules/tests/fixtures/locked_installs/relock_bad/projects/two/pyproject.toml b/scripts/rules/tests/fixtures/locked_installs/relock_bad/projects/two/pyproject.toml new file mode 100644 index 000000000..86e056a1b --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/relock_bad/projects/two/pyproject.toml @@ -0,0 +1,3 @@ +[project] +name = "two" +version = "0" diff --git a/scripts/rules/tests/fixtures/locked_installs/relock_good/.github/scripts/open-pr.sh b/scripts/rules/tests/fixtures/locked_installs/relock_good/.github/scripts/open-pr.sh new file mode 100644 index 000000000..21c42517e --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/relock_good/.github/scripts/open-pr.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +gh pr create --fill diff --git a/scripts/rules/tests/fixtures/locked_installs/relock_good/.github/workflows/update.yml b/scripts/rules/tests/fixtures/locked_installs/relock_good/.github/workflows/update.yml new file mode 100644 index 000000000..22f20317e --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/relock_good/.github/workflows/update.yml @@ -0,0 +1,10 @@ +name: Update +on: + schedule: + - cron: "0 15 * * *" +jobs: + update: + runs-on: ubuntu-latest + steps: + - run: make update + - run: .github/scripts/open-pr.sh diff --git a/scripts/rules/tests/fixtures/locked_installs/relock_good/Makefile b/scripts/rules/tests/fixtures/locked_installs/relock_good/Makefile new file mode 100644 index 000000000..07b1bfb71 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/relock_good/Makefile @@ -0,0 +1,4 @@ +update: + @for pyproject in libs/*/pyproject.toml projects/*/pyproject.toml; do \ + (cd "$$(dirname $$pyproject)" && uv lock --upgrade); \ + done diff --git a/scripts/rules/tests/fixtures/locked_installs/relock_good/libs/one/pyproject.toml b/scripts/rules/tests/fixtures/locked_installs/relock_good/libs/one/pyproject.toml new file mode 100644 index 000000000..52b7b2adf --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/relock_good/libs/one/pyproject.toml @@ -0,0 +1,3 @@ +[project] +name = "one" +version = "0" diff --git a/scripts/rules/tests/fixtures/locked_installs/relock_good/projects/two/pyproject.toml b/scripts/rules/tests/fixtures/locked_installs/relock_good/projects/two/pyproject.toml new file mode 100644 index 000000000..86e056a1b --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/relock_good/projects/two/pyproject.toml @@ -0,0 +1,3 @@ +[project] +name = "two" +version = "0" diff --git a/scripts/rules/tests/fixtures/locked_installs/test_bad/.github/workflows/ci.yml b/scripts/rules/tests/fixtures/locked_installs/test_bad/.github/workflows/ci.yml new file mode 100644 index 000000000..8cf92a7a3 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/test_bad/.github/workflows/ci.yml @@ -0,0 +1,8 @@ +name: CI +on: pull_request +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: uv sync --frozen --extra test + - run: python -m pytest tests/ diff --git a/scripts/rules/tests/fixtures/locked_installs/test_bad/Makefile b/scripts/rules/tests/fixtures/locked_installs/test_bad/Makefile new file mode 100644 index 000000000..a49b44c38 --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/test_bad/Makefile @@ -0,0 +1,2 @@ +test: + cd app && coverage run -m pytest diff --git a/scripts/rules/tests/fixtures/locked_installs/test_good/.github/workflows/ci.yml b/scripts/rules/tests/fixtures/locked_installs/test_good/.github/workflows/ci.yml new file mode 100644 index 000000000..3deeac9cb --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/test_good/.github/workflows/ci.yml @@ -0,0 +1,8 @@ +name: CI +on: pull_request +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: uv sync --frozen --extra test + - run: uv run --frozen python -m pytest tests/ diff --git a/scripts/rules/tests/fixtures/locked_installs/test_good/Makefile b/scripts/rules/tests/fixtures/locked_installs/test_good/Makefile new file mode 100644 index 000000000..d9f499d2f --- /dev/null +++ b/scripts/rules/tests/fixtures/locked_installs/test_good/Makefile @@ -0,0 +1,2 @@ +test: + cd app && uv run --frozen coverage run -m pytest diff --git a/scripts/rules/tests/test_load.py b/scripts/rules/tests/test_load.py index a25b017c2..7224fca20 100644 --- a/scripts/rules/tests/test_load.py +++ b/scripts/rules/tests/test_load.py @@ -123,7 +123,8 @@ def test_every_problem_is_reported_in_order(self) -> None: "Fix: write [[rule.exception]].", "[ex.layers.malformed] schema: unknown key 'extra' in " "ex.layers.malformed.exception. Fix: remove it or use one of " - "['expires', 'path', 'reason'].", + "['category', 'expires', 'issue', 'match', 'path', 'reason', " + "'removal'].", f"{MALFORMED}.exception.expires must be a date. " "Fix: write it as YYYY-MM-DD.", f"{MALFORMED}.downgrade needs a 'reason'. " diff --git a/scripts/rules/tests/test_locked_installs.py b/scripts/rules/tests/test_locked_installs.py new file mode 100644 index 000000000..201ee628d --- /dev/null +++ b/scripts/rules/tests/test_locked_installs.py @@ -0,0 +1,396 @@ +"""Tests for the locked-installs guard: lock, install and test from uv.lock.""" + +from __future__ import annotations + +import subprocess +import tempfile +import unittest +from pathlib import Path + +from scripts.rules.load import load_text +from scripts.rules.locked_installs import ( + INSTALL_RULE, + LOCK_RULE, + RELOCK_RULE, + REPO_ROOT, + TEST_RULE, + find_violations, + lock_violations, + relock_violations, + scan, + tracked_files, +) + +FIXTURES = Path(__file__).parent / "fixtures" / "locked_installs" +REGISTRY_HEADER = """ +schema_version = 1 +repo = "example" + +[[category]] +id = "dependencies" +title = "Dependencies" +""" +RULE_TEMPLATE = """ +[[rule]] +id = "{rule_id}" +category = "dependencies" +statement = "Installs come from uv.lock." +rationale = "docs/x.md" +scope = ["**"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "quality-guard" +name = "locked-installs" +""" + + +def registry_with(exceptions: str = "", rule_id: str = INSTALL_RULE) -> object: + text = REGISTRY_HEADER + RULE_TEMPLATE.format(rule_id=rule_id) + exceptions + registry, violations = load_text(text) + assert violations == [], violations + return registry + + +def violation_messages(root: Path, exceptions: str = "") -> list[str]: + return [ + violation.message + for violation in find_violations(root, registry_with(exceptions)) + if violation.rule_id == INSTALL_RULE + ] + + +class TestLockedInstallsProof(unittest.TestCase): + def test_rule_proof_install_from_lock(self) -> None: + bad = scan(FIXTURES / "install_bad") + good = scan(FIXTURES / "install_good") + + self.assertEqual( + sorted(site.text for site in bad.unlocked_installs), + [ + "pip install policyengine", + "pip install requests", + "pip_install(policyengine-us)", + "uv sync --extra test", + ], + ) + self.assertEqual(good.unlocked_installs, []) + + def test_rule_proof_test_from_lock(self) -> None: + bad = scan(FIXTURES / "test_bad") + good = scan(FIXTURES / "test_good") + + self.assertEqual( + sorted(site.text for site in bad.unlocked_tests), + ["coverage run -m pytest", "python -m pytest tests/"], + ) + self.assertEqual(good.unlocked_tests, []) + + def test_rule_proof_lock_committed(self) -> None: + bad_root = FIXTURES / "lock_bad" + good_root = FIXTURES / "lock_good" + + self.assertEqual( + [ + violation.message + for violation in lock_violations(bad_root, tracked_files(bad_root)) + ], + [ + "app/pyproject.toml has no committed uv.lock", + "no .github/workflows/pr.yml job runs uv lock --check", + ], + ) + self.assertEqual(lock_violations(good_root, tracked_files(good_root)), []) + + def test_rule_proof_scheduled_relock(self) -> None: + bad_root = FIXTURES / "relock_bad" + good_root = FIXTURES / "relock_good" + + self.assertEqual( + [ + violation.message + for violation in relock_violations(bad_root, tracked_files(bad_root)) + ], + [".github/workflows/update.yml does not relock projects/two/uv.lock"], + ) + self.assertEqual(relock_violations(good_root, tracked_files(good_root)), []) + + +class TestCommandClassification(unittest.TestCase): + def scan_script(self, script: str) -> tuple[list[str], list[str]]: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + (root / "run.sh").write_text(script, encoding="utf-8") + findings = scan(root) + return ( + [site.text for site in findings.unlocked_installs], + [site.text for site in findings.unlocked_tests], + ) + + def test_frozen_and_locked_syncs_and_runs_are_locked(self) -> None: + installs, tests = self.scan_script( + "uv sync --frozen --extra test\n" + "uv sync --locked\n" + "uv run --frozen pytest tests/\n" + "uv run --locked python -m unittest\n" + "uv lock --upgrade\n" + ) + + self.assertEqual((installs, tests), ([], [])) + + def test_every_unlocked_installer_is_reported(self) -> None: + installs, _tests = self.scan_script( + "uv run python -m tool\n" + "uv run --frozen --with requests python x.py\n" + "uv pip install requests\n" + "uv tool install ruff\n" + "uvx ruff check\n" + "python -m pip install requests\n" + "pipx install poetry\n" + "uv add requests\n" + "poetry install\n" + ) + + self.assertEqual(len(installs), 9, installs) + + def test_continuations_and_chains_are_read_as_commands(self) -> None: + installs, tests = self.scan_script( + "cd app && uv sync \\\n --extra test && uv run pytest\n" + ) + + self.assertEqual(installs, ["uv sync --extra test", "uv run pytest"]) + self.assertEqual(tests, ["uv run pytest"]) + + def test_text_tools_and_shell_keywords_do_not_hide_or_invent_installs( + self, + ) -> None: + installs, _tests = self.scan_script( + "VERSION := $(shell sed -n 's/.*uv tool install ruff==\\(.*\\)/\\1/p' a)\n" + 'if [ -z "$V" ]; then echo "uv tool install ruff=="; fi\n' + 'name="$(uv run python -c pass)"\n' + "RUFF := uv tool run ruff@$(VERSION)\n" + ) + + self.assertEqual( + installs, + ["name=$(uv run python -c pass)", "RUFF := uv tool run ruff@$(VERSION)"], + ) + + def test_comments_and_echoed_instructions_are_not_commands(self) -> None: + installs, tests = self.scan_script( + '# uv sync --extra test\necho " uv sync"\nprintf "pip install x"\n' + ) + + self.assertEqual((installs, tests), ([], [])) + + def test_workflow_sites_are_reported_on_their_own_line(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + workflow = root / ".github" / "workflows" / "ci.yml" + workflow.parent.mkdir(parents=True) + workflow.write_text( + "name: CI\n" + "on: pull_request\n" + "jobs:\n" + " first:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: uv run python one.py\n" + " second:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: |\n" + " echo start\n" + " uv run python two.py\n", + encoding="utf-8", + ) + + findings = scan(root) + + self.assertEqual( + [str(site) for site in findings.unlocked_installs], + [ + ".github/workflows/ci.yml:7: uv run python one.py", + ".github/workflows/ci.yml:13: uv run python two.py", + ], + ) + + +class TestExceptions(unittest.TestCase): + def test_an_exception_allows_only_the_site_it_names(self) -> None: + exceptions = """ +[[rule.exception]] +path = ".github/workflows/ci.yml" +match = "pip install requests" +reason = "example" +removal = "never" +""" + messages = violation_messages(FIXTURES / "install_bad", exceptions) + + self.assertFalse(any("pip install requests" in m for m in messages)) + self.assertTrue(any("uv sync --extra test" in m for m in messages)) + + def test_an_exception_needs_a_site_and_an_end(self) -> None: + exceptions = """ +[[rule.exception]] +path = ".github/workflows/ci.yml" +reason = "no site and no end" +""" + messages = violation_messages(FIXTURES / "install_bad", exceptions) + + self.assertIn("exception for .github/workflows/ci.yml names no site", messages) + self.assertIn( + "exception for .github/workflows/ci.yml has no removal condition " + "and no review date with a tracking issue", + messages, + ) + + def test_a_dated_exception_with_an_issue_is_accepted(self) -> None: + exceptions = """ +[[rule.exception]] +path = "Dockerfile" +match = "pip install policyengine" +reason = "example" +expires = "2099-01-01" +issue = 1 +""" + messages = violation_messages(FIXTURES / "install_bad", exceptions) + + self.assertFalse(any("Dockerfile" in m for m in messages), messages) + + def test_a_pinned_tool_exception_must_pin_an_exact_version(self) -> None: + exceptions = """ +[[rule.exception]] +path = ".github/workflows/ci.yml" +match = "pip install requests" +category = "pinned-tool" +reason = "example" +""" + messages = violation_messages(FIXTURES / "install_bad", exceptions) + + self.assertIn( + "pinned-tool exception for .github/workflows/ci.yml does not pin an " + "exact version", + messages, + ) + + def test_a_tool_pinned_with_an_at_version_is_a_pinned_tool(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + (root / "Makefile").write_text("RUFF := uv tool run ruff@$(VERSION)\n") + exceptions = """ +[[rule.exception]] +path = "Makefile" +match = "uv tool run ruff@$(VERSION)" +category = "pinned-tool" +reason = "example" +""" + self.assertEqual(violation_messages(root, exceptions), []) + + def test_a_tool_at_a_moving_version_is_not_a_pinned_tool(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + (root / "Makefile").write_text("RUFF := uv tool run ruff@latest\n") + exceptions = """ +[[rule.exception]] +path = "Makefile" +match = "uv tool run ruff@latest" +category = "pinned-tool" +reason = "example" +""" + self.assertEqual( + violation_messages(root, exceptions), + ["pinned-tool exception for Makefile does not pin an exact version"], + ) + + def test_an_issue_without_a_review_date_is_not_an_end(self) -> None: + exceptions = """ +[[rule.exception]] +path = "Dockerfile" +match = "pip install policyengine" +reason = "example" +issue = 1 +""" + messages = violation_messages(FIXTURES / "install_bad", exceptions) + + self.assertIn( + "exception for Dockerfile has no removal condition and no review " + "date with a tracking issue", + messages, + ) + + def test_an_exception_that_matches_no_site_is_stale(self) -> None: + exceptions = """ +[[rule.exception]] +path = ".github/workflows/ci.yml" +match = "pip install gone" +reason = "example" +removal = "never" +""" + messages = violation_messages(FIXTURES / "install_good", exceptions) + + self.assertEqual( + messages, + [ + "exception for .github/workflows/ci.yml (pip install gone) " + "matches no site" + ], + ) + + +class TestRepository(unittest.TestCase): + def test_every_install_capable_file_is_scanned(self) -> None: + listed = subprocess.run( + ["git", "ls-files"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.splitlines() + expected = set() + for path in listed: + name = Path(path).name.lower() + if "fixtures" in Path(path).parts: + continue + text = (REPO_ROOT / path).read_text(encoding="utf-8", errors="replace") + if ( + path.startswith(".github/workflows/") + or name.endswith((".sh", ".mk")) + or name == "makefile" + or ("dockerfile" in name and not name.endswith(".dockerignore")) + or "compose" in name + or ( + name.endswith(".py") + and "tests" not in Path(path).parts + and "modal.Image" in text + ) + ): + expected.add(path) + + scanned = scan(REPO_ROOT).scanned + + self.assertEqual(expected - scanned, set()) + self.assertIn( + "projects/policyengine-simulation-gateway/src/" + "policyengine_simulation_gateway/app.py", + scanned, + ) + + def test_the_repository_has_no_unregistered_unlocked_install(self) -> None: + registry_text = (REPO_ROOT / "rules" / "registry.toml").read_text() + registry, problems = load_text(registry_text) + self.assertEqual(problems, []) + + violations = find_violations(REPO_ROOT, registry) + + self.assertEqual([str(violation) for violation in violations], []) + + def test_rule_ids_are_registered(self) -> None: + registry_text = (REPO_ROOT / "rules" / "registry.toml").read_text() + registry, _problems = load_text(registry_text) + rule_ids = {rule.id for rule in registry.rules} + + self.assertLessEqual( + {LOCK_RULE, INSTALL_RULE, TEST_RULE, RELOCK_RULE}, rule_ids + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/run_quality_guards.py b/scripts/run_quality_guards.py index 45dda356e..9c74f88b4 100644 --- a/scripts/run_quality_guards.py +++ b/scripts/run_quality_guards.py @@ -10,7 +10,7 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from scripts.rules import ci_scripts, pyright # noqa: E402 +from scripts.rules import ci_scripts, locked_installs, pyright # noqa: E402 from scripts.rules import guard as rules_guard # noqa: E402 Guard = tuple[str, Callable[[], list[str]]] @@ -19,6 +19,7 @@ ("rules-registry", rules_guard.check), ("ci-scripts", ci_scripts.check), ("pyright-coverage", pyright.check_coverage), + ("locked-installs", locked_installs.check), ) diff --git a/scripts/test-integration-local.sh b/scripts/test-integration-local.sh index 8fb2b892b..c3c93a344 100755 --- a/scripts/test-integration-local.sh +++ b/scripts/test-integration-local.sh @@ -37,8 +37,8 @@ check_service "simulation-executor" 8082 echo "" echo "Running integration tests (excluding GCP workflow tests)..." cd projects/policyengine-apis-integ -uv sync --extra test -uv run pytest tests/ -v -m "not requires_gcp" +uv sync --frozen --extra test +uv run --frozen pytest tests/ -v -m "not requires_gcp" cd ../.. echo ""