diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f77930088..27addfa07 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -369,12 +369,15 @@ updates: update-types: - version-update:semver-major - # ── uv (languages/python — the Python SDK's test dependencies) ── - # The package has no runtime dependency, so uv.lock holds only the `test` - # extra: pytest, attrs, Pydantic, SQLAlchemy and psycopg, and what they - # need. tests-python.yml installs it with `uv sync --locked`. The extra has - # lower bounds and no upper bounds, so a new minor or patch release changes - # only the lock. + # ── uv (languages/python — the Python SDK's test and build tools) ── + # The package has no runtime dependency. So uv.lock contains two sets of + # tools: the `test` extra (pytest, attrs, Pydantic, SQLAlchemy, psycopg and + # Alembic), and the `build` dependency group (maturin, the build backend), + # with what they need. `mise run build` installs both with + # `uv sync --locked`. Dependabot's uv parser reads `[dependency-groups]` + # as well as the extras, so maturin is updated with the rest. Both have + # lower bounds and no major upper bound, so a new minor or patch release + # changes only the lock. - package-ecosystem: uv directory: /languages/python # Monthly, matching the other non-npm toolchains. No `day:`, for the diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index 1573cefaf..13511e9ba 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -27,6 +27,9 @@ on: pull_request: paths: - languages/python/** + # The example has its own lock and its own tests, which CI does not + # run. A change only there must not start this workflow. + - '!languages/python/examples/**' # The crates the workspace compiles by path, in the order Cargo.lock # lists them. The last two are what stack-kms's `profile` feature pulls # in. The root manifest @@ -38,6 +41,9 @@ on: - packages/stack-auth/** - packages/stack-profile/** - Cargo.toml + # Cargo reads the root .cargo/config.toml for this workspace too. A + # [patch] or [build] change there changes what this workspace compiles. + - .cargo/** # The EQL codegen and the catalog it reads. The SDK's marker names are # generated from the catalog (#1385). A change to either must rerun the # suite that reads them. @@ -59,6 +65,9 @@ on: # Go workflow's eql-postgres job does, and runs the secrets pre-flight. - packages/eql/src/** - packages/eql/mise.toml + # The EQL build compiles eql-codegen, whose versions these two choose. + - packages/eql/Cargo.toml + - packages/eql/Cargo.lock - packages/eql/tasks/** - .github/actions/require-cs-secrets/** - .github/workflows/tests-python.yml @@ -206,15 +215,15 @@ jobs: live: name: "Python live: encrypt into PostgreSQL with EQL v4" # A fork's pull request has no secrets: skip it rather than fail it, as - # the integration workflows do. - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + # the integration workflows do. Dependabot's pull requests come from this + # repository but get no Actions secrets either, so they skip too. + if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' }} runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 45 - env: - CS_WORKSPACE_CRN: ${{ vars.CS_WORKSPACE_CRN }} - CS_CLIENT_ID: ${{ vars.CS_CLIENT_ID }} - CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }} - CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }} + # The CS_* credentials are given to the `Live tests` step alone, not the + # job. Every other step, the builds above all, runs third-party code: each + # dependency's build.rs and proc-macros, and maturin. None of it runs with + # the credentials set. services: postgres: image: postgres:17 @@ -268,22 +277,33 @@ jobs: # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs cache_key_prefix: mise-v1-ubuntu-2404 - # One cache per workspace: this one, and EQL's, whose eql-codegen - # writes the bundle. + # One step for both workspaces: this one, and EQL's, whose eql-codegen + # writes the bundle. Two steps would each prune the shared cargo + # registry to their own crates before they save, so each save would + # lose the other workspace's crates. - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: - workspaces: languages/python + workspaces: | + languages/python + packages/eql - - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - with: - workspaces: packages/eql + # The builds, with no credentials: the extension into .venv, and the + # EQL v4 bundle. + - name: Build the extension and the EQL bundle + run: | + mise run build + mise run build:eql-bundle - # Builds the wheel and the bundle, installs the bundle, and runs the - # live tests; a skip fails the task. + # Installs the bundle and runs the live tests, building nothing; a skip + # fails the task. The only step with the credentials. - name: Live tests env: + CS_WORKSPACE_CRN: ${{ vars.CS_WORKSPACE_CRN }} + CS_CLIENT_ID: ${{ vars.CS_CLIENT_ID }} + CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }} + CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }} EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres sslmode=disable" - run: mise run test:live + run: mise run test:live:run # One abi3-py39 wheel per platform the FFI ships for, by the same Rust # triples. python-ci.test.mjs holds the list to the protect-ffi platform @@ -439,9 +459,14 @@ jobs: # wheel names its engine the same way. The checkout is clean here, so # the `-dirty` check the script skips with the override has nothing to # add. python-ci.test.mjs holds the variable name to build.rs. + # On a pull request, HEAD is GitHub's test merge commit, which is on no + # branch. The pull request's head commit is the one a bug report can + # name, so the wheel records that one. - name: Read the engine commit id: engine - run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + env: + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: echo "commit=$COMMIT" >> "$GITHUB_OUTPUT" # `--locked`: the Cargo.lock in the tree is the one built, as in every # other check of this workspace. `--out dist` puts the wheel where the @@ -459,6 +484,9 @@ jobs: target: ${{ matrix.target }} manylinux: ${{ matrix.manylinux }} rust-toolchain: ${{ steps.rust.outputs.version }} + # A fixed maturin, inside pyproject.toml's range. Without it the + # action downloads the newest release on every run. + maturin-version: v1.16.0 docker-options: -e STACKENCRYPT_ENGINE_COMMIT # The build time is the number this job exists to record. The spec's @@ -508,6 +536,8 @@ jobs: set -euo pipefail python -m pip install --no-index --no-deps languages/python/dist/*.whl python -c 'import stackencrypt as s; print("stackencrypt", s.__version__, "engine", s.__engine__)' + # A wheel must not carry the deterministic key source. + python -c 'from stackencrypt import _engine; assert not hasattr(_engine, "_use_deterministic_keys"), "this wheel has the deterministic-kms test feature"' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: @@ -530,6 +560,8 @@ jobs: wheels-summary: name: one wheel per platform, build time per platform needs: [wheels] + # Run when a leg fails too, so the table still shows the legs that built. + if: ${{ !cancelled() }} runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: diff --git a/AGENTS.md b/AGENTS.md index 5e52f8ed8..f25cbc044 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin, never in a crate manifest, so `cargo test` links libpython. `pyproject.toml` sets it, and `test:deterministic` names it again in its `--features` list; `scripts/__tests__/python-ci.test.mjs` holds the two lists together. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It also runs one test binary of `crates/core`, `deterministic_switch`, with the `deterministic-kms` feature: the switch is for the whole process, so that test needs a process of its own. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so the crates that decide what a stored value is resolve to the versions the root tests: every `vitaminc` crate, `recipher`, `ore-rs`, `clww-ore`, `cts-common` and `zerokms-protocol`. `scripts/__tests__/python-lock-parity.test.mjs` compares those crates in the two locks, by version, source and checksum. Other shared crates may differ. To re-seed, copy the root `Cargo.lock` over it and run `cargo metadata` there. That command prunes the copy to what the workspace uses. It can also move a crate to a newer version, when this workspace turns on a feature the root does not. Move each such crate back with `cargo update -p @ --precise `, until the parity test passes. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs the live tests: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It builds the extension and the bundle in a step with no credentials, then runs `mise run test:live:run`, which builds nothing, in the only step that has them. So no dependency's build script runs with the credentials set. Locally, `mise run test:live` does both. It skips on a fork's pull request, and the task fails on any skipped test. A third job, `wheels`, builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. It also locks maturin, the build backend, in the `build` dependency group. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. It then builds the package with `--no-build-isolation`, through the locked maturin in `.venv`, so uv downloads no build tool outside the lock. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. The `wheels` job builds with `PyO3/maturin-action` instead, with maturin's version fixed in the workflow. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `languages/python/examples/customer-directory`: An internal FastAPI example of the Python SDK, written twice: with psycopg (`app.py`) and with SQLAlchemy (`app_sqlalchemy.py`). It has its own `pyproject.toml` and `uv.lock`, and installs `stackencrypt` from this checkout. Its tests need real credentials, and CI does not run them. diff --git a/languages/python/Cargo.lock b/languages/python/Cargo.lock index f1746dd78..6a522d95f 100644 --- a/languages/python/Cargo.lock +++ b/languages/python/Cargo.lock @@ -412,9 +412,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.43" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "num-traits", @@ -455,7 +455,7 @@ dependencies = [ "subtle", "thiserror 1.0.69", "vitaminc-ore", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", "zeroize", ] @@ -1351,7 +1351,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", "tokio", "tower-service", "tracing", @@ -1537,15 +1537,14 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.3.4" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2", + "socket2 0.5.10", "widestring", - "windows-registry", - "windows-result", - "windows-sys 0.61.2", + "windows-sys 0.48.0", + "winreg", ] [[package]] @@ -1557,6 +1556,16 @@ dependencies = [ "serde", ] +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -1702,9 +1711,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "leb128fmt" @@ -1982,12 +1991,12 @@ dependencies = [ "rand 0.10.1", "subtle-ng", "thiserror 1.0.69", - "vitaminc-context 0.8.1", + "vitaminc-context 0.8.0", "vitaminc-hmac", "vitaminc-ore", "vitaminc-prf", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] @@ -2236,7 +2245,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -2275,7 +2284,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] @@ -2444,22 +2453,22 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 3.0.6", ] [[package]] @@ -2752,9 +2761,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.17.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ "core-foundation-sys", "libc", @@ -2962,6 +2971,16 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -3003,8 +3022,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc 0.8.1", - "vitaminc-protected 0.8.1", + "vitaminc 0.8.0", + "vitaminc-protected 0.8.0", "web-time", "zeroize", "zerokms-protocol", @@ -3030,15 +3049,15 @@ dependencies = [ "stack-kms", "thiserror 1.0.69", "uuid", - "vitaminc 0.8.1", - "vitaminc-aead 0.8.1", + "vitaminc 0.8.0", + "vitaminc-aead 0.8.0", "vitaminc-aead-frame", - "vitaminc-encrypt 0.8.1", + "vitaminc-encrypt 0.8.0", "vitaminc-hmac", "vitaminc-normalize", "vitaminc-ore", "vitaminc-prf", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", "zeroize", ] @@ -3076,8 +3095,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc 0.8.1", - "vitaminc-protected 0.8.1", + "vitaminc 0.8.0", + "vitaminc-protected 0.8.0", "zeroize", "zerokms-protocol", ] @@ -3345,7 +3364,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", "windows-sys 0.61.2", ] @@ -3431,9 +3450,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.11" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", "bitflags", @@ -3443,13 +3462,13 @@ dependencies = [ "http", "http-body", "http-body-util", + "iri-string", "pin-project-lite", "tokio", "tokio-util", "tower", "tower-layer", "tower-service", - "url", ] [[package]] @@ -3709,16 +3728,16 @@ dependencies = [ [[package]] name = "vitaminc" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd347492314ce981cd9fba0361e07d240e4e4a2e12e0107a7b86cd16abb9f8aa" +checksum = "ea37168a603d605732d0bf0672586139ab91736f767d7b85e173c84dcee19054" dependencies = [ - "vitaminc-aead 0.8.1", - "vitaminc-context 0.8.1", - "vitaminc-encrypt 0.8.1", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", - "vitaminc-traits 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-context 0.8.0", + "vitaminc-encrypt 0.8.0", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", + "vitaminc-traits 0.8.0", ] [[package]] @@ -3738,18 +3757,18 @@ dependencies = [ [[package]] name = "vitaminc-aead" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e723c592529da633fe7ba94d95904a9b6832e7995fc3f1d0da17bdbf8587c7a1" +checksum = "e9bb97fd543ee6a2b168e257ec69d0d274a775b1eaa3f856b5ac2941b01891c1" dependencies = [ "bytes", "futures-util", "serde", "thiserror 2.0.18", - "vitaminc-aead-derive 0.8.1", - "vitaminc-context 0.8.1", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", + "vitaminc-aead-derive 0.8.0", + "vitaminc-context 0.8.0", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] @@ -3766,9 +3785,9 @@ dependencies = [ [[package]] name = "vitaminc-aead-derive" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "553187b663af2a88f50cddc6fc4b5e58f045aa5f7296075cf33a56c972ff6b2b" +checksum = "0845a44c07ddc882ed47a0e92e04fc83496220f054b589ad0cbd215c5009e5b7" dependencies = [ "proc-macro2", "quote", @@ -3777,13 +3796,12 @@ dependencies = [ [[package]] name = "vitaminc-aead-frame" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a65349bb75a9ecc9fd4f057e5a6aac964bea68e57d03d55118a22455002a424" +checksum = "3b0299777255171712a296d59164691dc1e93d10bddaab6e248fede1790dfd2e" dependencies = [ "serde", - "vitaminc-aead 0.8.1", - "vitaminc-protected 0.8.1", + "vitaminc-aead 0.8.0", "zeroize", ] @@ -3799,12 +3817,12 @@ dependencies = [ [[package]] name = "vitaminc-context" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0179df09c5d8cab1c9d9f104eb807812851d1a36f97212542fb61f29e01646e4" +checksum = "a630079ba2d50680ee5048addd430eda30613559809dd6d383551e6d3d299826" dependencies = [ "mutants", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", ] [[package]] @@ -3823,72 +3841,72 @@ dependencies = [ [[package]] name = "vitaminc-encrypt" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265383e1184382f22570fa5be60db8b4f6b2321c21a2c5bf08a858705c4c8828" +checksum = "db45e073f92ae8ca27f8fe14270bc34c75e767559c2347098dacca95180d1ddd" dependencies = [ "aes-gcm", "aws-lc-rs", - "vitaminc-aead 0.8.1", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] [[package]] name = "vitaminc-hmac" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a244886bff4915e15000a69f5a10f14410cabb30c445e3d585057c06a9ce1a" +checksum = "b5fa0ae8bbad85ae30fb5c91b0b72b694d59bf2c2c1e7ad799eda66fdbd67e7c" dependencies = [ "hmac", "sha2 0.11.0", "vitaminc-prf", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", "zeroize", ] [[package]] name = "vitaminc-normalize" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d820d507ca7cce81772e38c1277b26acc422a4fd99402e6d9d49c412a5a135f0" +checksum = "f5a2d8875e0b8b35b7a4f9dd698bbd13c989837cea564b5a4cbb5371b78fa937" dependencies = [ "thiserror 2.0.18", "unicode-general-category", "unicode-normalization", - "vitaminc-aead 0.8.1", - "vitaminc-protected 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-protected 0.8.0", ] [[package]] name = "vitaminc-ore" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b80bf6789c1e2e1c192a255a9e2f6c3e7ffbce2fcb61d60edecee6ffc2f71ef" +checksum = "f05518b4d35cb0c592dad59afb11b8e883ea252449b9ae47ff5ddeff0ed49f75" dependencies = [ "thiserror 2.0.18", "unicode-general-category", "unicode-normalization", - "vitaminc-aead 0.8.1", - "vitaminc-context 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-context 0.8.0", "vitaminc-normalize", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", "zeroize", ] [[package]] name = "vitaminc-prf" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86cfcbfb4089f5bf334723cbf106f7d0d4709abf0ad12aaf49d6cf89f3e78de" +checksum = "916688478273ca3e5c6805b355534ce9263688391cd4ab23b08bef760039d325" dependencies = [ "mutants", "thiserror 2.0.18", - "vitaminc-aead 0.8.1", - "vitaminc-context 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-context 0.8.0", "vitaminc-normalize", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", ] [[package]] @@ -3910,9 +3928,9 @@ dependencies = [ [[package]] name = "vitaminc-protected" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81704a2c5bb14d108dd4aa20b4124d4b0c47a2695fc6330131cf0897350737aa" +checksum = "a6edc22562a9bbff0ba9704a2c43670b2c7cb1637524c045c3ba6436ed00917f" dependencies = [ "bitvec", "digest 0.11.3", @@ -3921,7 +3939,7 @@ dependencies = [ "serde_bytes", "subtle", "thiserror 2.0.18", - "vitaminc-protected-derive 0.8.1", + "vitaminc-protected-derive 0.8.0", "zeroize", ] @@ -3938,9 +3956,9 @@ dependencies = [ [[package]] name = "vitaminc-protected-derive" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eacd4b9ad7ae5196ee117944325aead3508da45d4c7ad73f682e64d62b09bb7" +checksum = "a7cce3a7d0ce6f57c0e82345f20e87f3737cccd3a70aec2a2ab49f68a3a5e24a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -3965,16 +3983,16 @@ dependencies = [ [[package]] name = "vitaminc-random" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9bc57a6532bfa847fa5093c2c39bdc333d1c0b724f982cf3c240e8b091d491" +checksum = "8bb253f783ce6f31a55a92f5a9f0526109b8caa6a465edb94244fd344a078c11" dependencies = [ "chacha20", "getrandom 0.4.2", "rand 0.10.1", "thiserror 2.0.18", - "vitaminc-protected 0.8.1", - "vitaminc-random-derives 0.8.1", + "vitaminc-protected 0.8.0", + "vitaminc-random-derives 0.8.0", "zeroize", ] @@ -3991,9 +4009,9 @@ dependencies = [ [[package]] name = "vitaminc-random-derives" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73e700b3a8e01ee5130939fbdc4dadb38bc4d4160c5f1d5f7fd139bab0491dfa" +checksum = "df0b3615936d6dded4b775856f4b0fa229d745d183f0b3754ac80035f16c27d9" dependencies = [ "proc-macro2", "quote", @@ -4018,17 +4036,17 @@ dependencies = [ [[package]] name = "vitaminc-traits" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a090c4d967d90bc3c609b73a3629e27ca752e0a5a62a5b00e0ea91f04617c60d" +checksum = "1e9098ee4b13446ef959583ee2f3c384df571b453179104f7b4960ffddf65ee9" dependencies = [ "anyhow", "bytes", "rmp-serde", "serde", "thiserror 2.0.18", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] @@ -4288,17 +4306,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -4326,6 +4333,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -4368,6 +4384,21 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -4407,6 +4438,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -4425,6 +4462,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -4443,6 +4486,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -4473,6 +4522,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -4491,6 +4546,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -4509,6 +4570,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -4527,6 +4594,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -4548,6 +4621,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/languages/python/README.md b/languages/python/README.md index f8380dbec..6b979ff1d 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -81,12 +81,19 @@ with psycopg.connect(DATABASE_URL) as conn: users = cipher.decrypt(cur.fetchall()) # a list of User ``` -`decrypt` builds whole `User` instances, so the `SELECT` must name every declared column, passthrough columns included. A row without one raises `stackencrypt.DecryptError` with the code `decrypt_missing_column`. Other columns in the row are ignored. +`decrypt` returns one `User` for each row, in the order of the rows. `decrypt` builds whole `User` instances, so the `SELECT` must name every declared column, passthrough columns included. A row without one raises `stackencrypt.DecryptError` with the code `decrypt_missing_column`. Other columns in the row are ignored. A passthrough `datetime` field needs a `timestamptz` column. psycopg reads a `timestamp` column as a naive `datetime`, with no `tzinfo`. `decrypt` refuses a naive `datetime` with `value_naive_datetime`, as `encrypt` does. `decrypt` checks every row before it sends anything. A value stored under another table or column raises `DecryptError` with the code `decrypt_foreign_row`. A passthrough value of another type than the field declares raises `InvalidValueError`. When any row fails, `decrypt` returns no instances. +Each of these errors names the row and the field, such as `row 317, field "email"`. Two errors name the field only. The engine raises them. It does not report the row yet: + +- `decrypt_malformed`, for a value whose JSON is not the field's EQL type. Its `i` names this table and column. No key request is made. +- `decrypt_kind_mismatch`, for a value that opens but was sealed as another type. This comes after the key requests. + +For these two, look for the bad value in that field's column, across all the rows of the call. + ## A field that can be missing Declare a field that can be `None` with `Optional` inside the `Annotated`: @@ -210,7 +217,7 @@ Each class must be declared with `stackencrypt.model` and mapped by SQLAlchemy. - `declaration_table_mismatch`: the table in `stackencrypt.model` is not the `__tablename__`. - `declaration_column_name`: an encrypted field's column has another name than the field, as with `mapped_column("email_address")`. - `declaration_column_type`: an encrypted field's column is not the EQL type of its marker. Set `type_annotation_map = eql_type_map` on the base. -- `declaration_unique_column`: an encrypted column has `unique=True` or a unique constraint. Each EQL value contains new random ciphertext, so the constraint would never find a duplicate. Use a unique index on `eql_v4.eq_term(col)`, as in "Migrations with Alembic". +- `declaration_unique_column`: an encrypted column has `unique=True` or a unique constraint. Each EQL value contains new random ciphertext, so the constraint would never find a duplicate. Equal values have equal `eq` terms, so use a unique index on `eql_v4.eq_term(col)`, as in "Migrations with Alembic". - `declaration_already_configured`: another class is configured for the same table. A statement that puts an encrypted value inside its SQL, such as `insert(Customer).values(email=...)`, raises `DeclarationError` with `declaration_inline_value`. The value is not stored. Pass the values as parameters instead, as `session.add` and `session.execute(insert(Customer), rows)` do. @@ -319,6 +326,8 @@ Nothing else is hidden. These all contain the plaintext: - a log line that formats a field, such as `log.info(user.email)` - `dataclasses.asdict`, Pydantic's `model_dump`, `vars`, and `__dict__` - `pickle.dumps(user)`, and anything a debugger reads from the attributes +- a Pydantic `ValidationError`, which shows the rejected input. Set `hide_input_in_errors=True` in the model's `model_config` to leave it out. +- FastAPI's 422 response, which repeats the input of each error. `hide_input_in_errors` does not change it. Replace FastAPI's handler for `RequestValidationError` with one that leaves out `input`. If the class defines its own `__repr__`, `__str__`, or `__rich_repr__`, the decorator refuses it. `__rich_repr__` is the method the `rich` library prints through. To keep your own methods, pass `redact=False` to `stackencrypt.model`. Your methods must then hide the plaintext themselves. @@ -335,6 +344,8 @@ A model field cannot be a `uuid.UUID`. The decorator refuses it with the code `d - Leave the key column out of the model. `encrypt` never writes a column the model does not declare, and `decrypt` ignores one. Insert and select the key yourself, beside the encrypted values. - Declare the field as `Annotated[str, Passthrough]`. Convert with `str(key)` before `encrypt`. Select the column as `id::text`, because psycopg reads a `uuid` column as a `uuid.UUID`, and `decrypt` refuses that for a `str` field. +A key the database generates, such as an `identity` column, cannot be a required field: the model is encrypted before PostgreSQL creates the key. Leave it out of the model, or declare it `Annotated[Optional[int], Ignore] = None`. Insert with `RETURNING id` to get the new key. To read it back, select it beside the encrypted columns. `decrypt` returns the instances in the order of the rows, so the key of row `n` belongs to instance `n`. With the SQLAlchemy layer, declare it `Mapped[Annotated[int, Ignore]]` with `mapped_column(primary_key=True)`, as in that section's example. + ## What is not supported yet - Range queries with `between`. Use `gte` and `lte` together. diff --git a/languages/python/crates/core/build.rs b/languages/python/crates/core/build.rs index aadc701ed..21908a4f8 100644 --- a/languages/python/crates/core/build.rs +++ b/languages/python/crates/core/build.rs @@ -8,15 +8,22 @@ //! was not built from. The scope is every stack-* crate, not only //! `stack-encrypt`: it depends by path on `stack-kms` and //! `stack-encrypt-derive`, and a wider scope can only add a `-dirty` that is -//! true. Edits outside `packages/stack-*` (this workspace, the Python -//! package) are not engine edits and do not mark the engine dirty; the wheel -//! version and the checkout commit describe those. +//! true. Two more files choose the engine's code: the root `Cargo.toml`, +//! which sets the versions of the crates the engine takes with +//! `workspace = true`, and this workspace's `Cargo.lock`, which holds the +//! exact versions. An edit to either also marks the engine dirty. Other +//! edits outside `packages/stack-*` (this workspace, the Python package) are +//! not engine edits; the wheel version and the checkout commit describe +//! those. //! //! `STACKENCRYPT_ENGINE_COMMIT` in the environment wins over git, for a build //! from a source tree that is not a checkout. It is trusted as given: with no //! checkout there is nothing to check it against. With neither, the build //! fails: an "unknown" engine in a wheel is a report nobody can act on. +#[path = "engine_commit_rules.rs"] +mod engine_commit_rules; + use std::env; use std::path::{Path, PathBuf}; use std::process::Command; @@ -26,19 +33,28 @@ const ENV_OVERRIDE: &str = "STACKENCRYPT_ENGINE_COMMIT"; /// `stack-*` entry in it is engine source for the `-dirty` check. const PACKAGES_DIR: &str = "../../../../packages"; const ENGINE_PREFIX: &str = "stack-"; +/// The files outside the stack-* crates that choose the engine's dependency +/// versions, relative to this crate. +const DEPENDENCY_FILES: [&str; 2] = ["../../../../Cargo.toml", "../../Cargo.lock"]; fn main() { println!("cargo:rerun-if-env-changed={ENV_OVERRIDE}"); - let commit = match env::var(ENV_OVERRIDE) { - Ok(value) if !value.is_empty() => value.to_ascii_lowercase(), - _ => commit_from_git(), + let given = env::var(ENV_OVERRIDE) + .ok() + .and_then(|value| engine_commit_rules::from_override(&value)); + let (commit, source) = match given { + Some(commit) => (commit, "env"), + None => (commit_from_git(), "git"), }; - let sha = commit.strip_suffix("-dirty").unwrap_or(&commit); assert!( - sha.len() == 40 && sha.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')), + engine_commit_rules::is_valid(&commit), "the engine commit must be a full 40-character hex SHA (optionally `-dirty`), got {commit:?}" ); println!("cargo:rustc-env={ENV_OVERRIDE}={commit}"); + // Where the commit came from, for the test that checks it against git. + // cargo also sets the variable above when it runs the tests, so the + // test cannot tell an override from the environment. + println!("cargo:rustc-env=STACKENCRYPT_ENGINE_COMMIT_SOURCE={source}"); } fn commit_from_git() -> String { @@ -77,6 +93,12 @@ fn commit_from_git() -> String { for dir in engine_dirs(&manifest_dir) { println!("cargo:rerun-if-changed={}", dir.display()); } + for file in DEPENDENCY_FILES { + println!( + "cargo:rerun-if-changed={}", + manifest_dir.join(file).display() + ); + } let commit = git(&["rev-parse", "HEAD"]).unwrap_or_else(|| { panic!( @@ -97,9 +119,15 @@ fn commit_from_git() -> String { dirs.iter() .map(|d| format!(":(exclude){}/fuzz", d.display())), ); + paths.extend(DEPENDENCY_FILES.iter().map(|file| (*file).to_owned())); let mut status = vec!["status", "--porcelain", "--"]; status.extend(paths.iter().map(String::as_str)); - let dirty = git(&status).is_some_and(|s| !s.is_empty()); + // A status that fails cannot say the tree is clean, so it fails the + // build, as `rev-parse` does. + let porcelain = git(&status).unwrap_or_else(|| { + panic!("`git status` failed, so the build cannot tell whether the engine has edits") + }); + let dirty = !porcelain.is_empty(); if dirty { format!("{commit}-dirty") } else { diff --git a/languages/python/crates/core/engine_commit_rules.rs b/languages/python/crates/core/engine_commit_rules.rs new file mode 100644 index 000000000..59784a67f --- /dev/null +++ b/languages/python/crates/core/engine_commit_rules.rs @@ -0,0 +1,16 @@ +//! The rules for the engine commit that `build.rs` records. `build.rs` +//! includes this file with `mod`, and the crate's tests include it with +//! `#[path]`, so the rules are tested without running a build. + +/// The commit given in `STACKENCRYPT_ENGINE_COMMIT`, in lower case, or +/// `None` when the value is empty: an empty value means "read git". +pub fn from_override(value: &str) -> Option { + (!value.is_empty()).then(|| value.to_ascii_lowercase()) +} + +/// Whether `commit` is a full 40-character lower-case hex SHA, with +/// `-dirty` after it or not. +pub fn is_valid(commit: &str) -> bool { + let sha = commit.strip_suffix("-dirty").unwrap_or(commit); + sha.len() == 40 && sha.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-date b/languages/python/crates/core/fuzz/corpus/value_frame/valid-date index 15c46c861..239847fc8 100644 Binary files a/languages/python/crates/core/fuzz/corpus/value_frame/valid-date and b/languages/python/crates/core/fuzz/corpus/value_frame/valid-date differ diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-decimal b/languages/python/crates/core/fuzz/corpus/value_frame/valid-decimal index dcaf90462..90fdd51d5 100644 Binary files a/languages/python/crates/core/fuzz/corpus/value_frame/valid-decimal and b/languages/python/crates/core/fuzz/corpus/value_frame/valid-decimal differ diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-float32 b/languages/python/crates/core/fuzz/corpus/value_frame/valid-float32 index bd4f5bd17..c28e3f482 100644 Binary files a/languages/python/crates/core/fuzz/corpus/value_frame/valid-float32 and b/languages/python/crates/core/fuzz/corpus/value_frame/valid-float32 differ diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-int64 b/languages/python/crates/core/fuzz/corpus/value_frame/valid-int64 index 7084678f4..54e8eb424 100644 Binary files a/languages/python/crates/core/fuzz/corpus/value_frame/valid-int64 and b/languages/python/crates/core/fuzz/corpus/value_frame/valid-int64 differ diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-text b/languages/python/crates/core/fuzz/corpus/value_frame/valid-text index 77324ca53..341f8995f 100644 Binary files a/languages/python/crates/core/fuzz/corpus/value_frame/valid-text and b/languages/python/crates/core/fuzz/corpus/value_frame/valid-text differ diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-timestamp b/languages/python/crates/core/fuzz/corpus/value_frame/valid-timestamp index 0395bb4b9..85a976842 100644 Binary files a/languages/python/crates/core/fuzz/corpus/value_frame/valid-timestamp and b/languages/python/crates/core/fuzz/corpus/value_frame/valid-timestamp differ diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-true b/languages/python/crates/core/fuzz/corpus/value_frame/valid-true index fc2b5693e..a903574af 100644 Binary files a/languages/python/crates/core/fuzz/corpus/value_frame/valid-true and b/languages/python/crates/core/fuzz/corpus/value_frame/valid-true differ diff --git a/languages/python/crates/core/fuzz/fuzz_targets/value_frame.rs b/languages/python/crates/core/fuzz/fuzz_targets/value_frame.rs index fd1a3f948..49ba3764f 100644 --- a/languages/python/crates/core/fuzz/fuzz_targets/value_frame.rs +++ b/languages/python/crates/core/fuzz/fuzz_targets/value_frame.rs @@ -14,17 +14,24 @@ // engine returns does. It must never panic. A scalar it accepts that the // write check also accepts must write back as a passthrough and read as // itself. +// +// The first input byte chooses the branch. 0x00 is Read, and the rest of the +// input is the frame, byte for byte, so a seed in `corpus/value_frame/` is +// 0x00 then a real frame. Any other byte is Write, and the rest decodes +// through `arbitrary`. `arbitrary` itself cannot carry a frame as it is: its +// `Vec` puts a byte of its own before each element. -use arbitrary::Arbitrary; +use arbitrary::{Arbitrary, Unstructured}; use libfuzzer_sys::fuzz_target; use stack_encrypt::dynamic::ValueFrame; use stackencrypt_core::values::{self, Kind, Scalar, Typed, Value}; +/// The first input byte that chooses Read; any other byte chooses Write. +const READ: u8 = 0x00; + +/// A Write input: a kind and a host value. #[derive(Arbitrary, Debug)] -enum Input { - Write(MirrorKind, Host), - Read(Vec), -} +struct Write(MirrorKind, Host); /// `Option`: `Passthrough` is `None`. #[derive(Arbitrary, Debug, Clone, Copy)] @@ -140,42 +147,55 @@ fn same(a: &Scalar, b: &Scalar) -> bool { } } -fuzz_target!(|input: Input| { - match input { - Input::Write(kind, host) => { - let (kind, value) = (kind.kind(), host.value()); - if values::scalar(kind, value).is_err() { - return; - } - let frame = ValueFrame::of(Typed(kind, value)).expect("an accepted value frames"); - let read = values::read(frame.risky_ref()).expect("a written frame reads"); - assert!( - same(&read, &expected(kind, value)), - "{read:?} for {kind:?} {value:?}" - ); - if let Some(plain) = independent(kind, value) { - assert_eq!(frame.risky_ref(), plain.risky_ref(), "{kind:?} {value:?}"); - } - } - Input::Read(bytes) => { - let Ok(read) = values::read(&bytes) else { - return; - }; - let Some(value) = read.value() else { - return; - }; - // `values::read` range-checks a timestamp, but not a date or a - // decimal: it reads any `i32` day count, and any scale and - // mantissa. `values::scalar` refuses a date outside the engine's - // range, a scale over 28, and a mantissa over 96 bits. The engine - // only returns a passthrough the SDK wrote, so read accepting - // more than write is not a defect. - if values::scalar(None, value).is_err() { - return; - } - let frame = ValueFrame::of(Typed(None, value)).expect("a read scalar writes back"); - let again = values::read(frame.risky_ref()).expect("a written frame reads"); - assert!(same(&again, &read), "{again:?} after {read:?}"); - } +fuzz_target!(|data: &[u8]| { + let Some((&branch, rest)) = data.split_first() else { + return; + }; + if branch == READ { + read_back(rest); + } else if let Ok(Write(kind, host)) = Write::arbitrary_take_rest(Unstructured::new(rest)) { + write_back(kind, host); } }); + +/// Write: an accepted value frames, reads back as itself narrowed to its +/// kind, and, for a plain Rust type, gives the bytes `ValueFrame::of` gives +/// that type. +fn write_back(kind: MirrorKind, host: Host) { + let (kind, value) = (kind.kind(), host.value()); + if values::scalar(kind, value).is_err() { + return; + } + let frame = ValueFrame::of(Typed(kind, value)).expect("an accepted value frames"); + let read = values::read(frame.risky_ref()).expect("a written frame reads"); + assert!( + same(&read, &expected(kind, value)), + "{read:?} for {kind:?} {value:?}" + ); + if let Some(plain) = independent(kind, value) { + assert_eq!(frame.risky_ref(), plain.risky_ref(), "{kind:?} {value:?}"); + } +} + +/// Read: `bytes` never panic `values::read`, and a scalar that both checks +/// accept writes back as a passthrough and reads as itself. +fn read_back(bytes: &[u8]) { + let Ok(read) = values::read(bytes) else { + return; + }; + let Some(value) = read.value() else { + return; + }; + // `values::read` range-checks a timestamp, but not a date or a decimal: + // it reads any `i32` day count, and any scale and mantissa. + // `values::scalar` refuses a date outside the engine's range, a scale + // over 28, and a mantissa over 96 bits. The engine only returns a + // passthrough the SDK wrote, so read accepting more than write is not a + // defect. + if values::scalar(None, value).is_err() { + return; + } + let frame = ValueFrame::of(Typed(None, value)).expect("a read scalar writes back"); + let again = values::read(frame.risky_ref()).expect("a written frame reads"); + assert!(same(&again, &read), "{again:?} after {read:?}"); +} diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index 3d87b20f4..b7c14a09f 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -3,7 +3,9 @@ //! Every `Cipher` in the process uses the same `StackCipher`. It is built on //! the first operation that needs it, by whichever thread gets there first, //! and never rebuilt: credentials are read once, and the default keyset is -//! loaded once (the spec in #1341, "Operations"). Beside it is one +//! loaded once (the spec in #1341, "Operations"). A child that `fork` made +//! after the client was built cannot use it, and is refused, as described at +//! [`shared`]. Beside it is one //! multi-thread tokio runtime. An operation hands its future to that runtime //! and blocks its own thread, with the GIL released, so calls from several //! Python threads run inside the engine at the same time. @@ -15,16 +17,15 @@ use std::collections::BTreeMap; use std::future::Future; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::{Mutex, MutexGuard, OnceLock}; use stack_auth::AutoStrategy; use stack_encrypt::kms::{ - ClientOpts, ConnectionInitError, EnvKeyProvider, FallbackKeyProvider, HttpConnection, - HttpConnectionOpts, KeyProvider, KeyProviderError, ProfileStore, StackKms, - StackKmsBuilderError, ZeroKMSConnection, ZeroKMSConnectionInit, ZeroKmsEndpoint, + endpoint_from_env, ClientOpts, ConnectionInitError, HttpConnection, HttpConnectionOpts, + KeyProvider, StackKms, ZeroKMSConnection, ZeroKMSConnectionInit, ZeroKmsEndpoint, }; -use stack_encrypt::StackCipher; +use stack_encrypt::{client_key_provider, StackCipher}; use zerokms_protocol::{ViturRequest, ViturRequestError}; use crate::credentials; @@ -40,6 +41,9 @@ pub struct Client { } static CLIENT: OnceLock = OnceLock::new(); +/// The ID of the process that built `CLIENT`. Set before `CLIENT`, so a +/// thread that sees the client sees its process ID. +static BUILT_IN: AtomicU32 = AtomicU32::new(0); // Serialises the build, so two threads making their first call at once build // one client and load the keyset once. `OnceLock::get_or_init` would do the // same, but it cannot fail, and a failed build must not be cached: the user @@ -58,17 +62,47 @@ fn locked(mutex: &Mutex) -> MutexGuard<'_, T> { .unwrap_or_else(|poisoned| poisoned.into_inner()) } +/// The client, if this process built it. A refusal if another process +/// built it: this process is a child that `fork` made after the build. +/// +/// The child's copy of the client cannot work: its runtime has no threads, +/// because a fork copies only the thread that called it. A new runtime is +/// not safe in the child either: tokio's locks keep a process-wide table of +/// waiting threads, and the child's copy lists the parent's threads. A new +/// runtime there blocked in 8 of 15 runs on Linux. So the child is refused. +/// No lock is taken here, so the refusal cannot block. +fn current() -> Result, Refusal> { + let Some(client) = CLIENT.get() else { + return Ok(None); + }; + if BUILT_IN.load(Ordering::Acquire) == std::process::id() { + return Ok(Some(client)); + } + Err(Refusal { + code: "stack_forked".to_owned(), + message: "this process was forked after the engine client was built, and a forked \ + process cannot use it. Make the first encrypt, decrypt or query call after \ + the fork: with Gunicorn, do not use --preload; with multiprocessing, use \ + the spawn start method." + .to_owned(), + field: None, + reason: None, + }) +} + /// The shared client, built on the first call. /// /// # Errors /// -/// The refusal of whatever failed: the credentials (`stack_auth_*`), the -/// client key or the ZeroKMS client (`stack_kms_*`), or the keyset load -/// (`stack_encrypt_*`). Nothing is cached on failure. +/// `stack_forked` in a child that `fork` made after the client was built +/// (see [`current`]). Otherwise the refusal of whatever failed: the +/// credentials (`stack_auth_*`), the client key or the ZeroKMS client +/// (`stack_kms_*`), or the keyset load (`stack_encrypt_*`). Nothing is +/// cached on failure. pub fn shared() -> Result<&'static Client, Refusal> { #[cfg(feature = "deterministic-kms")] refuse_after_switch()?; - if let Some(client) = CLIENT.get() { + if let Some(client) = current()? { return Ok(client); } let _building = locked(&BUILDING); @@ -76,11 +110,12 @@ pub fn shared() -> Result<&'static Client, Refusal> { // it, and the live client must then not be built. #[cfg(feature = "deterministic-kms")] refuse_after_switch()?; - if let Some(client) = CLIENT.get() { + if let Some(client) = current()? { return Ok(client); } let client = build()?; BUILDS.fetch_add(1, Ordering::Relaxed); + BUILT_IN.store(std::process::id(), Ordering::Release); Ok(CLIENT.get_or_init(|| client)) } @@ -92,16 +127,24 @@ pub fn shared() -> Result<&'static Client, Refusal> { /// after the switch. The two have different key source types, so a function /// that returned "the current client" would still leave each operation to /// write a branch per client. +/// +/// Both branches give the macro's value: the test branch leaves the block, +/// not the function, so code after `run!` runs in the test build too. macro_rules! run { (|$cipher:ident| $future:expr) => {{ - #[cfg(feature = "deterministic-kms")] - if let Some((cipher, runtime)) = $crate::client::deterministic() { - let $cipher = cipher; - return runtime.block_on($future); - } - let client = $crate::client::shared()?; - let $cipher = client.cipher(); - client.block_on($future) + // The label is used only in the test build. + #[allow(unused_labels)] + let value = 'run: { + #[cfg(feature = "deterministic-kms")] + if let Some((cipher, runtime)) = $crate::client::deterministic() { + let $cipher = cipher; + break 'run runtime.block_on($future); + } + let client = $crate::client::shared()?; + let $cipher = client.cipher(); + client.block_on($future) + }; + value }}; } pub(crate) use run; @@ -236,15 +279,14 @@ fn build() -> Result { async fn init() -> Result, Refusal> { // First: it reads one variable and sends nothing, so a bad value is // refused before the credentials are read. - let endpoint = endpoint_from_env()?; + // The endpoint and the client key come from the engine's own readers, + // the ones StackCipherBuilder uses, so the two cannot drift apart. + let endpoint = endpoint_from_env().map_err(|error| Refusal::of(&error))?; let credentials = AutoStrategy::detect().map_err(|error| credentials::refusal(&error))?; - let client_key = FallbackKeyProvider::new( - EnvKeyProvider, - ProfileClientKey(ProfileStore::resolve(None).ok()), - ) - .client_key() - .await - .map_err(|error| Refusal::of(&error))?; + let client_key = client_key_provider() + .client_key() + .await + .map_err(|error| Refusal::of(&error))?; // Requests go out one at a time, as the spec states the request count: // ceil(values / 500) key requests "in sequence". The Go guest sets the // same. @@ -260,38 +302,6 @@ async fn init() -> Result, Refusal> { .map_err(|error| Refusal::of(&error)) } -/// The ZeroKMS endpoint override, as `StackKmsBuilder` reads it: the first -/// of `CS_ZEROKMS_HOST` and the legacy `CS_VITUR_HOST` that is set. `None` -/// leaves the endpoint to the token's services claim, as every other SDK -/// does. `init` builds the client itself to count its requests, so it reads -/// the variable here; the builder's own reader is private. -fn endpoint_from_env() -> Result, Refusal> { - for env_var in stack_encrypt::kms::vars::CS_ZEROKMS_HOST { - if let Ok(value) = std::env::var(env_var) { - return value.parse().map(Some).map_err(|source| { - Refusal::of(&StackKmsBuilderError::InvalidEndpoint { env_var, source }) - }); - } - } - Ok(None) -} - -/// The CLI's profile as a key provider, where it resolves. Where it does -/// not, the error says what to do, as stack-encrypt's own provider does. -struct ProfileClientKey(Option); - -impl KeyProvider for ProfileClientKey { - async fn client_key(&self) -> Result { - match &self.0 { - Some(store) => store.client_key().await, - None => Err(KeyProviderError::NotConfigured( - "no client key: set CS_CLIENT_ID and CS_CLIENT_KEY, or log in with `stash auth login`" - .into(), - )), - } - } -} - /// The HTTP connection, counting every request it sends. pub struct Counting(HttpConnection); diff --git a/languages/python/crates/core/src/credentials.rs b/languages/python/crates/core/src/credentials.rs index a0d496a4c..5bae0296d 100644 --- a/languages/python/crates/core/src/credentials.rs +++ b/languages/python/crates/core/src/credentials.rs @@ -49,8 +49,11 @@ pub fn refusal(error: &AuthError) -> Refusal { /// no variable at all. `None` keeps the engine's message and help. fn message(error: &AuthError) -> Option { let text = match error { - AuthError::NotAuthenticated(_) => "no credentials were found. Set CS_CLIENT_ACCESS_KEY \ - and CS_WORKSPACE_CRN, or log in with `stash auth login`." + // Also raised for a profile that was found but lacks its region or + // client ID, so the message does not say that nothing was found. + AuthError::NotAuthenticated(_) => "no usable credentials were found: none are set, or \ + the profile from `stash auth login` is incomplete. Set CS_CLIENT_ACCESS_KEY and \ + CS_WORKSPACE_CRN, or log in again with `stash auth login`." .to_owned(), AuthError::MissingWorkspaceCrn(_) => { "CS_CLIENT_ACCESS_KEY is set and CS_WORKSPACE_CRN is not. Set both.".to_owned() diff --git a/languages/python/crates/core/src/lib.rs b/languages/python/crates/core/src/lib.rs index fd2faf5fe..a404dd6f9 100644 --- a/languages/python/crates/core/src/lib.rs +++ b/languages/python/crates/core/src/lib.rs @@ -24,6 +24,16 @@ compile_error!( "stackencrypt-core needs its `eql` feature: every build of the SDK holds the EQL types" ); +// The deterministic key source takes every key from a fixed seed. A wheel +// built with it would export `_use_deterministic_keys`, which switches every +// key in the process to that seed. Only the test build uses it, and that +// build is the dev profile. A release build with the feature is refused here, +// so a release wheel cannot carry it. +#[cfg(all(feature = "deterministic-kms", not(debug_assertions)))] +compile_error!( + "the `deterministic-kms` feature is for test builds only: a release build must not carry it" +); + // The EQL resolver: the Go guest's own file, compiled here too, so the two // SDKs run one resolver and cannot drift apart. It switches on this crate's // `eql` feature. Its tests run here as well as in the guest. `HOLDS_EQL` is @@ -44,9 +54,35 @@ pub fn engine_commit() -> &'static str { env!("STACKENCRYPT_ENGINE_COMMIT") } +#[cfg(test)] +#[path = "../engine_commit_rules.rs"] +mod engine_commit_rules; + #[cfg(test)] mod tests { use super::engine_commit; + use super::engine_commit_rules::{from_override, is_valid}; + + const SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + + /// The override rules: an empty value reads git, a value is lower-cased, + /// and only a full hex SHA, with `-dirty` or not, is a commit. + #[test] + fn the_override_is_lower_cased_and_checked() { + assert_eq!(from_override(""), None); + assert_eq!( + from_override(&SHA.to_ascii_uppercase()).as_deref(), + Some(SHA) + ); + assert!(is_valid(SHA)); + assert!(is_valid(&format!("{SHA}-dirty"))); + assert!(!is_valid(&SHA[..39])); + assert!(!is_valid(&format!("{SHA}0"))); + assert!(!is_valid(&SHA.replace('a', "g"))); + assert!(!is_valid(&SHA.to_ascii_uppercase())); + assert!(!is_valid(&format!("{SHA}-clean"))); + } + use std::process::Command; #[test] fn engine_commit_is_a_full_hex_sha() { @@ -58,4 +94,61 @@ mod tests { "{commit:?}" ); } + + /// `git` in this crate's directory, or `None` when it fails. + fn git(args: &[&str]) -> Option { + let output = Command::new("git") + .args(args) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output() + .ok()?; + output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).into_owned()) + } + + /// The commit is this checkout's, with `-dirty` exactly when a file + /// under a `packages/stack-*` crate has changed outside its `fuzz/`, or + /// the root `Cargo.toml` or this workspace's `Cargo.lock` has changed. + /// The status is read for wider paths and filtered here, so the test + /// does not repeat build.rs's pathspec. + #[test] + fn engine_commit_is_this_checkouts_commit() { + // A commit given in STACKENCRYPT_ENGINE_COMMIT is trusted as given. + if env!("STACKENCRYPT_ENGINE_COMMIT_SOURCE") != "git" { + return; + } + let Some(head) = git(&["rev-parse", "HEAD"]) else { + return; + }; + let status = git(&[ + "status", + "--porcelain", + "--", + "../../../../packages", + "../../../../Cargo.toml", + "../../Cargo.lock", + ]) + .expect("git status runs in a checkout"); + // A porcelain line is a two-letter status, a space and the path from + // the repository root. A rename shows "old -> new". + let dirty = status.lines().any(|line| { + line.get(3..).is_some_and(|paths| { + paths.split(" -> ").any(|path| { + let path = path.trim_matches('"'); + path == "Cargo.toml" + || path == "languages/python/Cargo.lock" + || path + .strip_prefix("packages/") + .and_then(|rest| rest.split_once('/')) + .is_some_and(|(krate, rest)| { + krate.starts_with("stack-") && !rest.starts_with("fuzz/") + }) + }) + }) + }); + let expected = format!("{}{}", head.trim(), if dirty { "-dirty" } else { "" }); + assert_eq!(engine_commit(), expected, "git status:\n{status}"); + } } diff --git a/languages/python/crates/core/src/query.rs b/languages/python/crates/core/src/query.rs index dee96490b..5ea210df0 100644 --- a/languages/python/crates/core/src/query.rs +++ b/languages/python/crates/core/src/query.rs @@ -177,11 +177,23 @@ pub fn answers(plan: &ModelPlan, field: &str, op: OpSpec) -> Result<(), Refusal> if operators(plan, field)?.contains(&op.name()) { return Ok(()); } + // The marker, as the Python check names it: the EQL type, or + // Passthrough for a field with none. + let marker = plan + .fields() + .iter() + .find(|declared| declared.name == field) + .map(|declared| { + declared + .eql + .as_ref() + .map_or_else(|| "Passthrough".to_owned(), |(name, _)| name.clone()) + }); Err(Refusal { code: "query_unsupported".to_owned(), message: format!("field {field:?} has no index that answers {:?}.", op.name()), field: Some(field.to_owned()), - reason: None, + reason: marker, }) } @@ -194,6 +206,18 @@ mod tests { use crate::decrypt::tests::{cipher, run, stored, users}; use crate::encrypt::{Column, Field}; + /// The refusal names the field's marker in `reason`, as the Python + /// check does: the EQL type, or `Passthrough`. + #[test] + fn query_unsupported_names_the_marker_in_reason() { + let refused = answers(&users(), "email", OpSpec::Lt).unwrap_err(); + assert_eq!(refused.code, "query_unsupported"); + assert_eq!(refused.reason.as_deref(), Some("TextEq")); + + let refused = answers(&users(), "age", OpSpec::Eq).unwrap_err(); + assert_eq!(refused.reason.as_deref(), Some("Passthrough")); + } + fn term(cipher: &StackCipher, email: &str) -> serde_json::Value { let json = run(query_with( &cipher.default_keyset(), diff --git a/languages/python/crates/core/src/refusal.rs b/languages/python/crates/core/src/refusal.rs index ed7b31fea..989fd4a40 100644 --- a/languages/python/crates/core/src/refusal.rs +++ b/languages/python/crates/core/src/refusal.rs @@ -18,11 +18,15 @@ pub struct Refusal { } impl Refusal { - /// The code and message of `error`, with no field or reason. + /// The code and message of `error`, with no field or reason. The + /// engine's help follows the message as a sentence of its own: most + /// engine messages end with no full stop. pub fn of(error: &dyn Diagnostic) -> Self { + let text = error.to_string(); let message = match error.help() { - Some(help) => format!("{error} {help}"), - None => error.to_string(), + Some(help) if text.ends_with(['.', '!', '?']) => format!("{text} {help}"), + Some(help) => format!("{text}. {help}"), + None => text, }; Self { code: error.code().map_or_else( @@ -44,7 +48,44 @@ pub fn python_code(engine_code: &str) -> String { #[cfg(test)] mod tests { - use super::python_code; + use super::{python_code, Refusal}; + + /// An engine error with a message and a help, written by hand: this + /// crate has no derive for errors. + #[derive(Debug)] + struct WithHelp { + message: &'static str, + } + + impl std::fmt::Display for WithHelp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.message) + } + } + + impl std::error::Error for WithHelp {} + + impl miette::Diagnostic for WithHelp { + fn code<'a>(&'a self) -> Option> { + Some(Box::new("stack_test::joined")) + } + + fn help<'a>(&'a self) -> Option> { + Some(Box::new("Do this.")) + } + } + + #[test] + fn the_message_and_its_help_are_two_sentences() { + let joined = Refusal::of(&WithHelp { + message: "the plan has no field", + }); + assert_eq!(joined.message, "the plan has no field. Do this."); + let ended = Refusal::of(&WithHelp { + message: "the plan has no field.", + }); + assert_eq!(ended.message, "the plan has no field. Do this."); + } #[test] fn engine_codes_replace_the_path_separator() { diff --git a/languages/python/crates/core/tests/deterministic_switch.rs b/languages/python/crates/core/tests/deterministic_switch.rs new file mode 100644 index 000000000..3809a4ea2 --- /dev/null +++ b/languages/python/crates/core/tests/deterministic_switch.rs @@ -0,0 +1,25 @@ +//! The switch to the deterministic client of the `deterministic-kms` test +//! build. A test binary is its own process, and the switch is for the whole +//! process, so these checks run in this binary alone. `test:cargo` runs it +//! with the feature on. + +#![cfg(feature = "deterministic-kms")] + +use stackencrypt_core::client; + +#[test] +fn after_the_switch_the_live_client_is_refused_and_a_second_switch_too() { + client::use_deterministic([7; 32]).expect("the first switch succeeds"); + assert!(client::deterministic().is_some()); + + // An operation that skipped the check for the deterministic client + // would send requests to ZeroKMS. It is refused instead. + let Err(refused) = client::shared() else { + panic!("the live client is refused"); + }; + assert_eq!(refused.code, "stack_test_build"); + assert_eq!(client::build_count(), 0); + + let refused = client::use_deterministic([7; 32]).expect_err("a second switch is refused"); + assert_eq!(refused.code, "stack_test_build"); +} diff --git a/languages/python/crates/core/tests/gate_frame_reader.rs b/languages/python/crates/core/tests/gate_frame_reader.rs index 9d1dbd42e..89dfa44ec 100644 --- a/languages/python/crates/core/tests/gate_frame_reader.rs +++ b/languages/python/crates/core/tests/gate_frame_reader.rs @@ -2,13 +2,16 @@ //! frame and recover each field's value kind, with only the public API of //! `stack-encrypt`'s `dynamic` module. //! -//! The record is built with `ValueFrame::of`, the same type and writer -//! `dynamic::decrypt` returns its opened records in. This test does not -//! decrypt: it proves the reading half, which is the half the extension owns. +//! The record is built with `ValueFrame::of`, the same type +//! `dynamic::decrypt` returns its opened records in. The writer is not the +//! same: `ValueFrame::of` writes through the value model's `Modeled` writer, +//! and `decrypt` writes each record with a plain `FrameWriter`. So this test +//! does not check the bytes of a real decrypt result. It does not decrypt: +//! it proves the reading half, which is the half the extension owns. -use stack_encrypt::dynamic::frame::Frame; +use stack_encrypt::dynamic::frame::{Frame, FrameWriter}; use stack_encrypt::dynamic::plaintext::{ - Encrypt, Error, FixedType, Int, Logical, MapSink, Physical, SeqSink, Sink, SourceError, + Encrypt, Error, FixedType, Int, Logical, MapSink, Physical, SeqSink, Sink, SourceError, Width, }; use stack_encrypt::dynamic::ValueFrame; @@ -30,6 +33,28 @@ impl Encrypt for User { } } +/// A record with the other shapes a field can have: a typed null, a +/// sequence and a nested map. The typed null is written as a plain null. +struct Shapes; + +impl Encrypt for Shapes { + fn visit(self, sink: S) -> Result { + let mut map = sink.map(Some(3))?; + let date = FixedType::new(Some(Logical::Date), Width::I32).expect("a date has a width"); + map.entry("left")?.typed_null(date)?; + let mut tags = map.entry("tags")?.seq(Some(2))?; + tags.element()?.scalar(None, Physical::Text("a"))?; + tags.element()?.scalar(None, Physical::Int(Int::I64(2)))?; + tags.end()?; + let mut address = map.entry("address")?.map(Some(1))?; + address + .entry("city")? + .scalar(None, Physical::Text("London"))?; + address.end()?; + map.end() + } +} + #[derive(Clone)] struct Nickname; @@ -92,8 +117,8 @@ impl Sink for Reader<'_> { Ok(()) } - fn typed_null(self, _: FixedType) -> Result<(), ReadError> { - self.scalar(None, Physical::Null) + fn typed_null(self, ty: FixedType) -> Result<(), ReadError> { + self.scalar(ty.logical(), Physical::Null) } fn seq(self, _: Option) -> Result { @@ -192,6 +217,75 @@ fn an_outside_sink_recovers_each_field_kind_from_a_record_frame() { ); } +#[test] +fn an_outside_sink_reads_a_typed_null_a_sequence_and_a_nested_map() { + let record = ValueFrame::of(Shapes).expect("the record is in the value model"); + let frame = Frame::parse(record.risky_ref()).expect("a ValueFrame is one frame"); + + assert_eq!( + read(frame), + vec![ + // A frame writer writes a typed null as the plain null: a + // frame is plaintext, so padding hides nothing in it (vitaminc's + // FrameWriter). A decrypted record's null carries no type, and + // the extension takes a field's type from its plan. + seen("left", None, "null", false), + seen("tags", None, "text", false), + seen("tags", None, "int", false), + seen("city", None, "text", false), + ], + ); +} + +/// A batch of two records, written the way `dynamic::decrypt` writes one +/// (`write_record` in stack-encrypt's `record.rs`): a plain `FrameWriter`, +/// a sequence, one map per record, and each field's `ValueFrame` walked into +/// its entry. The reader must see the same fields as in a `ValueFrame::of` +/// record, once for each record. +#[test] +fn an_outside_sink_reads_a_batch_written_as_decrypt_writes_it() { + let record = |email: &str, age: i64| { + vec![ + ("email", ValueFrame::of(email).expect("a str frames")), + ("age", ValueFrame::of(age).expect("an i64 frames")), + ] + }; + let rows = [ + record("ada@example.com", 36), + record("grace@example.com", 85), + ]; + + let mut bytes = Vec::new(); + let mut seq = FrameWriter::new(&mut bytes) + .seq(Some(rows.len())) + .expect("a sequence starts"); + for fields in rows { + let mut map = seq + .element() + .expect("an element") + .map(Some(fields.len())) + .expect("a map starts"); + for (name, value) in fields { + value + .visit(map.entry(name).expect("an entry")) + .expect("a field walks"); + } + map.end().expect("a map ends"); + } + seq.end().expect("the sequence ends"); + + let frame = Frame::parse(&bytes).expect("the batch is one frame"); + assert_eq!( + read(frame), + vec![ + seen("email", None, "text", false), + seen("age", None, "int", false), + seen("email", None, "text", false), + seen("age", None, "int", false), + ], + ); +} + #[test] fn malformed_frame_bytes_are_refused_before_any_walk() { let record = ValueFrame::of(User).expect("the record is in the value model"); diff --git a/languages/python/crates/core/tests/value_conformance.rs b/languages/python/crates/core/tests/value_conformance.rs index f06e9221c..4cf16513b 100644 --- a/languages/python/crates/core/tests/value_conformance.rs +++ b/languages/python/crates/core/tests/value_conformance.rs @@ -21,6 +21,31 @@ const CORPUS: &str = include_str!("../testdata/value-conformance.json"); /// (`int8`, the unsigned and 128-bit integers, `uuid`), it is not a scalar /// (`array`, `object`), or it is a timestamp in a unit other than the /// nanoseconds the SDK writes. +/// The vectors the SDK writes, in corpus order: every scalar kind it has, +/// from the corpus at this commit. A vector added on a refresh fails the +/// test with its name. +const WRITTEN: &[&str] = &[ + "false", + "true", + "int16", + "int32", + "int64", + "float32", + "float64", + "text", + "text-nul", + "bytes", + "date", + "date-year-zero", + "timestamp", + "timestamp-nanos", + "timestamp-expanded-year", + "timestamp-negative-year", + "decimal", + "decimal-min", + "decimal-small", +]; + const NOT_WRITTEN: &[&str] = &[ "null", "undefined", @@ -155,7 +180,7 @@ fn decimal(text: &str) -> (i128, u8) { #[test] fn every_vector_the_sdk_writes_gives_its_bytes_and_reads_back() { let corpus = corpus(); - let mut written = 0; + let mut written = Vec::new(); for vector in corpus["vectors"].as_array().expect("vectors") { let name = vector["name"].as_str().expect("a name"); let (mut text, mut bytes) = (String::new(), Vec::new()); @@ -185,10 +210,9 @@ fn every_vector_the_sdk_writes_gives_its_bytes_and_reads_back() { expected, "{name}: the read value" ); - written += 1; + written.push(name); } - // Every scalar kind the SDK has, from the corpus at this commit. - assert_eq!(written, 19); + assert_eq!(written, WRITTEN); } #[test] @@ -204,7 +228,7 @@ fn every_malformed_vector_is_refused() { #[test] fn every_normalized_vector_reads_as_its_canonical_value() { let corpus = corpus(); - let mut written = 0; + let mut written = Vec::new(); for vector in corpus["normalized"].as_array().expect("normalized") { let name = vector["name"].as_str().expect("a name"); let transport = unhex(vector["transport"].as_str().expect("transport")); @@ -245,10 +269,10 @@ fn every_normalized_vector_reads_as_its_canonical_value() { &reencode[..], "{name}: the written bytes" ); - written += 1; + written.push(name); } } } - // decimal-negative-zero, at this commit. - assert_eq!(written, 1); + // The normalized vectors that are not a typed null, at this commit. + assert_eq!(written, ["decimal-negative-zero"]); } diff --git a/languages/python/crates/engine/src/decrypt.rs b/languages/python/crates/engine/src/decrypt.rs index 0ab1fde14..9eb8c1036 100644 --- a/languages/python/crates/engine/src/decrypt.rs +++ b/languages/python/crates/engine/src/decrypt.rs @@ -9,6 +9,7 @@ //! only after every value has been turned into a Python object, so a call //! that fails returns no instance. +use pyo3::exceptions::{PyRecursionError, PyTypeError, PyUnicodeEncodeError, PyValueError}; use pyo3::prelude::*; use pyo3::sync::PyOnceLock; use pyo3::types::{PyBytes, PyDict, PyList, PyString}; @@ -130,7 +131,7 @@ fn stored_row(plan: &Plan, row: &Bound<'_, PyAny>, at: usize) -> PyResult { .eql(&plan.plan, field, text.as_bytes()) .map_err(in_row)?; } - None => passthrough(&mut stored, field, *kind, &value).map_err(in_row)?, + None => passthrough(&mut stored, field, *kind, &value)?.map_err(in_row)?, } } Ok(stored) @@ -146,7 +147,10 @@ fn stored_row(plan: &Plan, row: &Bound<'_, PyAny>, at: usize) -> PyResult { /// from 3.10. The text is then copied once more, into the engine's frame. /// /// The outer error is Python's own; the inner one is the refusal, for the -/// caller to name the row in. +/// caller to name the row in. Only an error that is about the value +/// becomes a refusal: a `str` with no UTF-8 encoding, or a `dict` that +/// `json.dumps` refuses. Any other error, such as `MemoryError` or +/// `KeyboardInterrupt`, propagates as it is. fn eql_text<'py>( field: &Field, value: &Bound<'py, PyAny>, @@ -161,7 +165,14 @@ fn eql_text<'py>( })) }; if let Ok(text) = value.cast::() { - return Ok(Ok(text.encode_utf8()?)); + // A lone surrogate is a str Python holds with no UTF-8 encoding. + return match text.encode_utf8() { + Ok(bytes) => Ok(Ok(bytes)), + Err(error) if error.is_instance_of::(py) => { + malformed("is a str that is not valid UTF-8".to_owned()) + } + Err(error) => Err(error), + }; } if let Ok(bytes) = value.cast::() { return match std::str::from_utf8(bytes.as_bytes()) { @@ -172,7 +183,14 @@ fn eql_text<'py>( if value.cast::().is_ok() { return match dumps(py)?.call1((value,)) { Ok(text) => Ok(Ok(text.cast::()?.encode_utf8()?)), - Err(_) => malformed("is a dict that is not JSON".to_owned()), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + malformed("is a dict that is not JSON".to_owned()) + } + Err(error) => Err(error), }; } if value.is_none() { @@ -185,38 +203,34 @@ fn eql_text<'py>( } /// A passthrough column's value, checked by exact type as on `encrypt`. +/// +/// An error Python raises while the value is read, such as one from a +/// user's `tzinfo`, propagates as it is, as on `encrypt`. The inner error +/// is the refusal. fn passthrough( stored: &mut Row, field: &Field, kind: PyKind, value: &Bound<'_, PyAny>, -) -> Result<(), Refusal> { +) -> PyResult> { let named = |refused: Refusal| Refusal { message: format!("field {:?}: {}", field.name, refused.message), field: Some(field.name.clone()), reason: Some("passthrough".to_owned()), ..refused }; - let read = match read(value, kind, "passthrough") { - Ok(Ok(read)) => read, - Ok(Err(refused)) => return Err(named(refused)), - Err(error) => { - return Err(named(Refusal { - code: "decrypt_malformed".to_owned(), - message: format!("the value could not be read: {error}"), - field: None, - reason: None, - })) - } + let read = match read(value, kind, "passthrough")? { + Ok(read) => read, + Err(refused) => return Ok(Err(named(refused))), }; - stored.passthrough(field, read.value()).map_err(|misfit| { + Ok(stored.passthrough(field, read.value()).map_err(|misfit| { named(Refusal { code: misfit.code().to_owned(), message: "the value is outside what the engine stores.".to_owned(), field: None, reason: None, }) - }) + })) } /// An opened value as the Python object of its field. diff --git a/languages/python/crates/engine/src/heap_hygiene.rs b/languages/python/crates/engine/src/heap_hygiene.rs index 1d84b7285..759ac1a63 100644 --- a/languages/python/crates/engine/src/heap_hygiene.rs +++ b/languages/python/crates/engine/src/heap_hygiene.rs @@ -133,11 +133,16 @@ unsafe impl GlobalAlloc for Spy { let trace = std::backtrace::Backtrace::force_capture().to_string(); // Linux writes `<…::CipherKey>::seal`, macOS `…::CipherKey::seal`. if trace.replace(['<', '>'], "").contains(UPSTREAM_SEAL) { - *UPSTREAM.lock().unwrap() += 1; + // A panic here, inside the allocator, would abort the whole + // test binary with no message, so a poisoned lock is used + // as it is, as GATE is. + *UPSTREAM + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) += 1; } else { FOUND .lock() - .unwrap() + .unwrap_or_else(|poisoned| poisoned.into_inner()) .push(format!("{} bytes freed unwiped:\n{trace}", layout.size())); } BUSY.with(|busy| busy.set(false)); diff --git a/languages/python/crates/engine/src/values.rs b/languages/python/crates/engine/src/values.rs index 4d24fec7e..79be62975 100644 --- a/languages/python/crates/engine/src/values.rs +++ b/languages/python/crates/engine/src/values.rs @@ -153,7 +153,7 @@ impl Read<'_> { /// # Errors /// /// `value_none` for `None`, `value_wrong_type` for another type, -/// `value_naive_datetime` for a datetime with no `tzinfo`, and the range +/// `value_naive_datetime` for a datetime with no UTC offset, and the range /// refusals for an `int` or `Decimal` the engine cannot hold. `field` is /// not set: the caller names it. pub fn read<'a>( @@ -197,7 +197,9 @@ pub fn read<'a>( other(Value::Date(ordinal - 719_163)) } PyKind::DateTime => { - if value.getattr("tzinfo")?.is_none() { + // Naive as Python defines it: no tzinfo, or a tzinfo whose + // utcoffset() is None. The subtraction below needs an offset. + if value.call_method0("utcoffset")?.is_none() { return Ok(Err(refusal( "value_naive_datetime", "is a naive datetime. Give it a tzinfo, such as timezone.utc", @@ -470,4 +472,97 @@ mod tests { fn decimal_of(value: &Bound<'_, PyAny>) -> Result, Refusal> { decimal(value, "decimal").unwrap() } + + /// A positive exponent multiplies the mantissa, and the sign applies to + /// it. A wrong sign or power would encrypt another number with no error. + #[test] + fn a_decimal_reads_as_its_exact_mantissa_and_scale() { + Python::initialize(); + Python::attach(|py| { + let decimal = py.import("decimal").unwrap().getattr("Decimal").unwrap(); + for (text, expected_mantissa, expected_scale) in [ + ("1.2E+3", 1_200, 0), + ("-0.05", -5, 2), + ("-7", -7, 0), + ("1E+28", 10_i128.pow(28), 0), + ] { + match decimal_of(&decimal.call1((text,)).unwrap()) { + Ok(Value::Decimal { mantissa, scale }) => { + assert_eq!( + (mantissa, scale), + (expected_mantissa, expected_scale), + "{text}" + ); + } + other => panic!("{text}: {other:?}"), + } + } + }); + } + + /// A date is days from 1970-01-01, and a datetime nanoseconds from the + /// epoch, in UTC. Both are checked against known numbers, before the + /// epoch and with a non-UTC offset, and read back as equal values. + #[test] + fn a_date_and_a_datetime_read_as_the_engines_numbers_and_back() { + Python::initialize(); + Python::attach(|py| { + let datetime = py.import("datetime").unwrap(); + let utc = datetime + .getattr("timezone") + .unwrap() + .getattr("utc") + .unwrap(); + let ten_hours = datetime + .getattr("timezone") + .unwrap() + .call1((datetime + .getattr("timedelta") + .unwrap() + .call1((0, 36_000)) + .unwrap(),)) + .unwrap(); + let day_before = datetime + .getattr("date") + .unwrap() + .call1((1969, 12, 31)) + .unwrap(); + let cases = [ + (day_before, PyKind::Date, Scalar::Date(-1)), + ( + // 1970-01-01T00:00:00.123456Z, written ten hours ahead. + datetime + .getattr("datetime") + .unwrap() + .call1((1970, 1, 1, 10, 0, 0, 123_456, &ten_hours)) + .unwrap(), + PyKind::DateTime, + Scalar::Timestamp(123_456_000), + ), + ( + datetime + .getattr("datetime") + .unwrap() + .call1((1969, 12, 31, 23, 59, 59, 999_999, &utc)) + .unwrap(), + PyKind::DateTime, + Scalar::Timestamp(-1_000), + ), + ]; + for (value, kind, scalar) in cases { + let read = read(&value, kind, "test").unwrap().unwrap(); + let got = match read.value() { + Value::Date(days) => Scalar::Date(i32::try_from(days).unwrap()), + Value::Timestamp(nanos) => Scalar::Timestamp(nanos), + other => panic!("{value}: read as {other:?}"), + }; + assert_eq!(got, scalar, "{value}"); + let opened = to_python(py, &scalar).unwrap().unwrap(); + assert!( + opened.bind(py).eq(&value).unwrap(), + "{value} came back as {opened}" + ); + } + }); + } } diff --git a/languages/python/docs/scorecard.md b/languages/python/docs/scorecard.md index fdd5a4bab..ed05fb74f 100644 --- a/languages/python/docs/scorecard.md +++ b/languages/python/docs/scorecard.md @@ -17,7 +17,7 @@ Each principle is marked: | G3. Run each check at the earliest stage the language allows | strained | | G4. One declaration serves the write, the query and the read | held | | G5. Batch by default | held | -| G6. A second way in has to earn its place | held | +| G6. A second way in has to earn its place | strained | | G7. State the design, and run its claims | held | | G8. Usage first, then reference | held | @@ -59,7 +59,9 @@ Each principle is marked: - Each query method is the single-value call the principle allows: one field, one search value. It makes no key request after the first call of the process. - A model's fields of different types go in one request. -## G6. A second way in has to earn its place: held +## G6. A second way in has to earn its place: strained + +Strained, not held: the two gaps below have no test. - There is one way to declare a model: `stackencrypt.model`. Every class library goes through it. `tests/_declaration_cases.py` declares one model with each of dataclasses, attrs, Pydantic, `NamedTuple`, and SQLAlchemy. - No test shows that each refusal fires for every one of the five libraries. diff --git a/languages/python/examples/customer-directory/FINDINGS.md b/languages/python/examples/customer-directory/FINDINGS.md new file mode 100644 index 000000000..113d1cb89 --- /dev/null +++ b/languages/python/examples/customer-directory/FINDINGS.md @@ -0,0 +1,137 @@ +# Findings from the customer directory example + +Measured on 9 November 2026, on one macOS machine, against real ZeroKMS. The SDK was a dev-profile build. Do not use these times as release-build figures. + +## Gaps in the SDK or its documentation + +Items 5, 6, and 7 were found before the SQLAlchemy layer existed. Items 1 to 7 each now have a status line. + +### 1. The README does not warn about Pydantic's `ValidationError` + +`Customer` is a Pydantic model that contains plaintext. By default, Pydantic prints the rejected input in a `ValidationError`. FastAPI's default 422 response also repeats the input: the example's first run returned `"input":"bad"`. + +The README's list of what still shows plaintext names `model_dump`, `vars`, `pickle`, and others. It does not name either of these. + +The example sets `hide_input_in_errors=True` and replaces FastAPI's 422 response. `tests/test_app.py` covers both. + +Suggested action: add both to the README's list, with the setting that hides the input. + +Status: addressed. The README's list in "What a printed model shows" now names both. It gives `hide_input_in_errors=True` for the `ValidationError`, and says that FastAPI's 422 response needs its own handler: with that setting, `errors()` still contains the input (checked with Pydantic 2.14.3). + +### 2. The README has no recipe for a database-generated key + +The README covers a UUID key only. A required integer `id` on the model cannot work, because the model is encrypted before PostgreSQL creates the `id`. + +The example leaves `id` out of the model. It works, but the model then cannot return the `id`. + +Suggested action: add the database-generated case to "Store a UUID key". + +Status: addressed. "Store a UUID key" now covers a key the database generates, for psycopg and for the SQLAlchemy layer. + +### 3. `decrypt` does not say whether it keeps row order + +Neither the `decrypt` docstring nor the README says that results come back in the order of the input rows. An application that needs a column outside the model must pair it with the decrypted result by position. + +The example avoids this by not returning `id`. I did not test the order. + +Suggested action: state the guarantee, and add a test for it. + +Status: addressed. The README and the `Cipher.decrypt` docstring state that the instances come back in the order of the rows. `tests_deterministic/test_order.py` checks it, with the rows in reverse order. + +### 4. The README does not cover uniqueness on an encrypted column + +The README shows only a hash index, and PostgreSQL cannot make a hash index unique. This B-tree index works: + +```sql +CREATE UNIQUE INDEX customers_email_unique ON customers (eql_v4.eq_term(email)); +``` + +A second insert with the same email raises `UniqueViolation` with SQLSTATE 23505. `EXPLAIN` shows an index scan on this index for `WHERE email = %s::eql_v4.query_text_eq`, with `enable_seqscan` off. + +Suggested action: document the unique index, and say that equal emails produce equal terms. + +Status: addressed. The SQLAlchemy section of the README explains `declaration_unique_column`. It says that equal values have equal `eq` terms, and shows a unique index on `eql_v4.eq_term(email)`. + +### 5. The README has no write or read path for SQLAlchemy + +The README shows how to declare a SQLAlchemy model, including relationships. It does not show an insert or a query. `app_sqlalchemy.py` needed three things that the README does not mention: + +- A custom column type. The tests fail with `sa.Text`, `sa.String`, or `sa.JSON` in its place. +- `sa.insert(Customer.__table__)` with the rows from `encrypt`, in place of `Session.add`. +- A `sa.cast` of the query term to `eql_v4.query_text_eq`. + +The `id` column works when it is marked `Ignore`. Search with these statements uses the unique index, as `EXPLAIN` shows with `enable_seqscan` off. + +Python ADR-0002 mentions a later SQLAlchemy layer. Until it exists, this is hand-written code for each application. + +Suggested action: add this recipe to the README, or say that the layer is planned. + +Status: addressed by the SQLAlchemy layer, `stackencrypt.sqlalchemy`. `app_sqlalchemy.py` now calls `configure_encryption([Customer])`. Its endpoints use `session.add` and `session.scalars`, with no SDK calls. The README section "Use it with the SQLAlchemy ORM" shows the write and the read. + +### 6. `Session.add` sends plaintext to PostgreSQL before it fails + +`Session.add` with a normal `Customer` object fails with a `DataError`, and no row is stored. But the plaintext has already reached the PostgreSQL server. The error text quotes the start of the value, even with `hide_parameters=True`: + +```text +DETAIL: Token "zed" is invalid. +CONTEXT: JSON data, line 1: zed... +``` + +The example logs only PostgreSQL's primary message, so this text is not in its log. An application that logs the exception would write part of the plaintext. A PostgreSQL server that logs errors could also record it. I did not check the server's log. + +Suggested action: warn about this in the README. The later SQLAlchemy layer could refuse this case before anything is sent. + +Status: addressed by the SQLAlchemy layer. Its write hook encrypts the values before SQLAlchemy sends the statement, so `Session.add` sends no plaintext. I did not check this example's PostgreSQL log again. + +### 7. A decrypted SQLAlchemy object is not connected to a session + +`select(Customer)` returns ciphertext dicts in `.email` and `.name`. `decrypt` returns new objects that SQLAlchemy reports as transient, with `id` set to `None`. So the usual pattern of loading an object, changing it, and committing it does not work. + +Suggested action: state this limit in the README. + +Status: addressed by the SQLAlchemy layer. A session query builds its objects as usual. The read hook then decrypts them in place with `set_committed_value`, so they are not marked as changed (ADR-0004). `tests/test_sqlalchemy_live.py` checks `session.dirty` after a load. + +### 8. Two `decrypt` errors from the engine do not name the row + +Most `decrypt` errors name the row and the field, such as `row 317, field "email"`. Two codes name the field only, when the engine finds the problem: + +- `decrypt_malformed`, for a value with an `i` that is not the field's EQL type. The engine refuses it before any key request. +- `decrypt_kind_mismatch`, for a value sealed as another type. The engine finds it after the value opens, so after the key requests. + +In a batch of 500 rows, the user must then search every row for the bad value. The Go SDK has the same gap: `Client.open` in `languages/golang/encrypt/records.go` returns the engine's error unchanged. Review thread 1509-F7 reported it for Python. + +The SDK could find the row for `decrypt_malformed` by running the engine's check again one row at a time. That needs a copy of every row's ciphertext on every call, and it cannot help with `decrypt_kind_mismatch` without new key requests. It would also make Python differ from Go. + +Suggested action: have the engine name the row in both refusals, so that Python and Go both get it. Issue #1508 already asks the engine to name the field in `decrypt_kind_mismatch`. Until then, state in the README and the `Cipher.decrypt` docstring that these two codes do not name the row. + +## Timings + +| Step | Time | +|---|---| +| `import app`, including `Cipher(Customer)` | 0.15 s | +| First `encrypt` of one customer, two runs | 0.95 s and 1.25 s | +| Next `encrypt` of one customer | about 0.22 s | +| `query` after the first call | 0.002 s | +| `psycopg.connect` and close | 0.011 s | +| 20 single-customer `encrypt` calls in sequence | 7.5 s | +| 20 single-customer `encrypt` calls on 20 threads | 0.73 s | + +The first request after start-up costs about 1 second more than later requests. The example has no warm-up call, as decided. + +## README claims that held + +- One `Cipher` can be shared between threads with no lock. The concurrency test passed, and the timings above show the calls overlap. +- `query` makes no noticeable request after the first call. It took 0.002 s. +- A Pydantic class works as both the FastAPI request body and the `stackencrypt` model. `default_factory` is accepted. +- `repr` shows `` for encrypted fields. +- Rows written by the psycopg app decrypt in the SQLAlchemy app, and the reverse. A test covers both directions. +- `decrypt` is all or nothing. One row stored under another table name made a whole list request fail with `decrypt_foreign_row`. +- Missing credentials stop the app at start-up, with `stack_auth_not_authenticated` and a message that says how to log in. + +## Limits of this example + +- It was not run on Python 3.9, the SDK's oldest supported version. It ran on 3.14 only. +- The statement that uv does not notice a Rust change is not tested. `mise run setup` always reinstalls the SDK. +- Dependabot does not update this example's `uv.lock`. +- CI does not run the tests, so the example can break with no CI failure. +- The tests print a Starlette warning that recommends `httpx2` in place of `httpx`. I left `httpx` in place, because I could not verify `httpx2`. diff --git a/languages/python/examples/customer-directory/README.md b/languages/python/examples/customer-directory/README.md index b769ab58c..218cb9b01 100644 --- a/languages/python/examples/customer-directory/README.md +++ b/languages/python/examples/customer-directory/README.md @@ -15,6 +15,12 @@ The SDK is not released, so its API can change. This example installs the SDK fr You need Docker, mise, and a Rust toolchain. mise installs the toolchain. +`mise run setup` builds EQL with EQL's own mise file, which installs its own Rust toolchain. Trust that file once, from this folder: + +```sh +mise trust ../../../../packages/eql +``` + You also need CipherStash credentials. Set `CS_CLIENT_ACCESS_KEY`, `CS_WORKSPACE_CRN`, `CS_CLIENT_ID`, and `CS_CLIENT_KEY`. Or log in with `stash auth login`. ## Run it diff --git a/languages/python/examples/customer-directory/app_sqlalchemy.py b/languages/python/examples/customer-directory/app_sqlalchemy.py index e0011bfe5..314329ff3 100644 --- a/languages/python/examples/customer-directory/app_sqlalchemy.py +++ b/languages/python/examples/customer-directory/app_sqlalchemy.py @@ -26,8 +26,10 @@ # hide_parameters leaves the bound values out of SQLAlchemy's error text. # PostgreSQL's own detail can still quote the start of a rejected value. +# make_url reads both postgresql:// and postgres://, as app.py accepts both. +# The URL object, not its str(), which writes the password as ***. engine = sa.create_engine( - os.environ["DATABASE_URL"].replace("postgresql://", "postgresql+psycopg://", 1), + sa.engine.make_url(os.environ["DATABASE_URL"]).set(drivername="postgresql+psycopg"), hide_parameters=True, ) diff --git a/languages/python/examples/customer-directory/mise.toml b/languages/python/examples/customer-directory/mise.toml index 609c5eee3..e9838414c 100644 --- a/languages/python/examples/customer-directory/mise.toml +++ b/languages/python/examples/customer-directory/mise.toml @@ -9,10 +9,11 @@ description = "Reset the example's database, install EQL v4 and the table, and i shell = "bash -c" run = """ set -euo pipefail -docker compose down --volumes -docker compose up --detach --wait # Built from the tree, because the released copy moves only at a version bump. +# Built first, so a failed build leaves the database as it was. mise -C ../../../../packages/eql run build +docker compose down --volumes +docker compose up --detach --wait psql_in_db() { docker compose exec -T db psql -U postgres -d customer_directory -v ON_ERROR_STOP=1 -q } diff --git a/languages/python/mise.toml b/languages/python/mise.toml index 70d4c393f..97ea93efe 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -26,9 +26,8 @@ set -euo pipefail # recreated, so a change to `[tools]` cannot leave the tests on the old one. # The `test` extra comes from uv.lock, with --locked, so a lock that does not # match pyproject.toml fails the build. Dependabot updates the lock. The -# package has no runtime dependency, so the lock fixes test tooling versions -# alone. maturin, the build backend, is not in the lock. uv resolves it from -# the index in an isolated build environment, within pyproject.toml's range. +# package has no runtime dependency, so the lock fixes tool versions alone: +# the test extra, and maturin, the build backend, in the `build` group. want="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')" have="$([ -x .venv/bin/python ] && .venv/bin/python -c 'import sys; print("%d.%d" % sys.version_info[:2])' || true)" if [ "$have" != "$want" ]; then @@ -40,13 +39,14 @@ fi # `wheels` job in tests-python.yml builds release, because the time it records # is the cost of the wheel a user would install. export MATURIN_PEP517_ARGS="${MATURIN_PEP517_ARGS:---profile dev}" -# The `test` extra at the locked versions. The project itself is left out: -# uv sync would install it editable. -uv sync --locked --extra test --no-install-project --python .venv/bin/python -# The package itself, built by maturin through the PEP 517 backend. Always -# reinstall it so a Rust change is picked up. Its dependencies are the ones -# above, so none is resolved here. -uv pip install --python .venv/bin/python --reinstall-package stackencrypt --no-deps . +# The `test` extra and the `build` group at the locked versions. The project +# itself is left out: uv sync would install it editable. +uv sync --locked --extra test --group build --no-install-project --python .venv/bin/python +# The package itself, built by the locked maturin in .venv through the PEP +# 517 backend. --no-build-isolation keeps uv from downloading a maturin of +# its own, outside the lock. Always reinstall it so a Rust change is picked +# up. Its dependencies are the ones above, so none is resolved here. +uv pip install --python .venv/bin/python --no-build-isolation --reinstall-package stackencrypt --no-deps . .venv/bin/python -c 'import stackencrypt as s; print("stackencrypt", s.__version__, "engine", s.__engine__)' """ @@ -85,22 +85,38 @@ libdir="$("$PYO3_PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("L export DYLD_FALLBACK_LIBRARY_PATH="${libdir}${DYLD_FALLBACK_LIBRARY_PATH:+:$DYLD_FALLBACK_LIBRARY_PATH}" export LD_LIBRARY_PATH="${libdir}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" cargo test --locked --workspace +# The deterministic switch is for the whole process, so its test is a binary +# of its own, built with the test feature. +cargo test --locked -p stackencrypt-core --features deterministic-kms --test deterministic_switch """ +[tasks."build:eql-bundle"] +description = "Build the EQL v4 bundle from the tree, for the live tests" +# Built from packages/eql, as the Go task go:test:postgres builds it: the +# copy under crates/eql-bindings/sql is the released bundle and moves only at +# the lockstep bump. +run = "mise -C ../../packages/eql run build" + [tasks."test:live"] -description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names, and run the live tests against it and ZeroKMS; needs the four CS_* credentials" -depends = ["build"] +description = "Build the extension and the EQL v4 bundle, then run test:live:run; needs the four CS_* credentials and EQL_TEST_DATABASE_URL" +depends = ["build", "build:eql-bundle"] +run = "mise run test:live:run" + +[tasks."test:live:run"] +description = "Install the EQL v4 bundle into the database EQL_TEST_DATABASE_URL names, and run the live tests against it and ZeroKMS; builds nothing" shell = "bash -c" -# The bundle is built from packages/eql, as the Go task go:test:postgres -# builds it: the copy under crates/eql-bindings/sql is the released bundle -# and moves only at the lockstep bump. The tests skip without credentials or -# a database, so a skip here is a failure: this task is where both are -# given. +# Builds nothing, so CI can run the builds in a step with no credentials and +# give the credentials to this task alone: a dependency's build script then +# never runs with them set (tests-python.yml's live job). It needs the +# extension in .venv and the bundle under packages/eql/release first; locally, +# run test:live, which builds both. The tests skip without credentials or a +# database, so a skip here is a failure: this task is where both are given. run = """ set -euo pipefail : "${EQL_TEST_DATABASE_URL:?set EQL_TEST_DATABASE_URL to a libpq connection string for a disposable PostgreSQL}" -mise -C ../../packages/eql run build -psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f ../../packages/eql/release/cipherstash-encrypt-v4.sql +bundle=../../packages/eql/release/cipherstash-encrypt-v4.sql +[ -f "$bundle" ] || { echo "error: $bundle is missing; run mise run build:eql-bundle first" >&2; exit 1; } +psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f "$bundle" log="${TMPDIR:-/tmp}/stackencrypt-live.log" .venv/bin/python -m pytest -m live -v -rs | tee "$log" if grep -q ' SKIPPED' "$log"; then diff --git a/languages/python/pyproject.toml b/languages/python/pyproject.toml index 191850a11..00a11c56c 100644 --- a/languages/python/pyproject.toml +++ b/languages/python/pyproject.toml @@ -35,6 +35,14 @@ test = [ "alembic>=1.13", ] +# maturin, the build backend, for `mise run build`. uv.lock records its +# version and hash, and Dependabot updates it with the test extra, after the +# same 7-day cooldown. The build installs it into .venv and builds with +# --no-build-isolation, so uv downloads no unlocked build tool. The range is +# [build-system]'s, which stays for an install by any other route. +[dependency-groups] +build = ["maturin>=1.9,<2.0"] + [tool.maturin] # The pure-Python package is the distribution; the Rust extension is its # `_engine` submodule, so the Python side can change without a recompile. diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index 000f6069b..1e6817ce6 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -52,6 +52,14 @@ class Cipher(Generic[T]): Every cipher in the process shares one engine client. It is built on the first ``encrypt`` or ``decrypt`` of any cipher, which loads the default keyset once. + + A process that ``os.fork`` made, as Gunicorn's workers are, cannot use a + client its parent built: the first ``encrypt``, ``decrypt`` or query + call must come after the fork. A call in such a child raises + ``StashError`` with the code ``stack_forked``. A fork before the first + call is fine: each child builds its own client. With Gunicorn, do not + use ``--preload``. With ``multiprocessing``, use the ``spawn`` start + method. """ def __init__(self, cls: Type[T]) -> None: @@ -73,7 +81,7 @@ def __init__(self, cls: Type[T]) -> None: # threads may build the same plan at once; either one is kept. self._row_plans: Dict[FrozenSet[str], object] = {} - def encrypt(self, models: Sequence[object]) -> List[Dict[str, object]]: + def encrypt(self, models: Sequence[T]) -> List[Dict[str, object]]: """Encrypt ``models``, instances of the declared class. Returns one dict per model, from column name to column value: the @@ -83,9 +91,23 @@ def encrypt(self, models: Sequence[object]) -> List[Dict[str, object]]: Every value is checked first, by exact type and range. A value that does not fit raises ``InvalidValueError`` naming the field, and no - request is made. An empty list makes no request at all. + request is made. A model that is not an instance of the declared + class raises ``InvalidValueError`` with the code + ``value_wrong_type``. An empty list makes no request at all. """ - return _engine.encrypt(self._plan, list(models)) + models = list(models) + for at, model in enumerate(models): + # The engine reads each field by name, so a model of another + # class with the same field names would be encrypted under this + # table. + if not isinstance(model, self._cls): + raise InvalidValueError( + "value_wrong_type", + f"model {at} is a {type(model).__name__}, not a " + f"{self._cls.__name__}. Encrypt only instances of the " + "declared class.", + ) + return _engine.encrypt(self._plan, models) def encrypt_rows(self, rows: Sequence[Mapping[str, object]]) -> List[Dict[str, object]]: """Encrypt ``rows``, mappings from field name to value, that may @@ -108,7 +130,7 @@ def encrypt_rows(self, rows: Sequence[Mapping[str, object]]) -> List[Dict[str, o raises ``InvalidValueError`` with the code ``value_row_shape``. No request is made when any row fails. The input rows are not changed. """ - rows = list(rows) + rows = _rows(rows, InvalidValueError, "value_row_shape") names = self._row_fields(rows, InvalidValueError, "value_row_shape", "Encrypt") if not names: return [dict(row) for row in rows] @@ -138,7 +160,7 @@ def decrypt_rows(self, rows: Sequence[Mapping[str, object]]) -> List[Dict[str, o fields than row 0, raises ``DecryptError`` with the code ``decrypt_row_shape``. No request is made when any row fails. """ - rows = list(rows) + rows = _rows(rows, DecryptError, "decrypt_row_shape") names = self._row_fields(rows, DecryptError, "decrypt_row_shape", "Decrypt") if not names: return [dict(row) for row in rows] @@ -196,8 +218,17 @@ def decrypt(self, rows: Sequence[Mapping[str, object]]) -> List[T]: and no request is made. A passthrough value of another type than the field's raises ``InvalidValueError``; it is never converted. The call is all or nothing: no instance is built when any row fails. + + It returns one instance for each row, in the order of the rows. + + Each of these errors names the row. Two errors from the engine name + the field only, not the row: ``decrypt_malformed`` for a value that + is not the field's EQL type, and ``decrypt_kind_mismatch`` for a + value sealed as another type. """ - return _engine.decrypt(self._plan, self._cls, list(rows)) + return _engine.decrypt( + self._plan, self._cls, _rows(rows, DecryptError, "decrypt_row_shape") + ) def eq(self, field: str, value: object) -> str: """The equality query term for ``value`` on the declared field @@ -399,6 +430,19 @@ def _not_decorated(cls: object) -> DeclarationError: ) +def _rows(rows: object, error: Type[StashError], code: str) -> list: + """``rows`` as a list. Raises ``error`` with ``code`` for one row on its + own, such as ``cursor.fetchone()`` returns: ``list()`` of a mapping is its + keys, and each key would be refused as a row that is not a mapping.""" + if isinstance(rows, Mapping): + raise error( + code, + "this takes a list of rows, not one row on its own. Pass [row], " + "or the result of fetchall().", + ) + return list(rows) # type: ignore[call-overload] + + def _build_plan(declaration: Declaration, fields: Sequence[DeclaredField]) -> object: """The engine's plan for ``fields`` of ``declaration``, in their order.""" return _engine.build_plan( diff --git a/languages/python/python/stackencrypt/_declaration.py b/languages/python/python/stackencrypt/_declaration.py index aba90b7d5..dac858fc7 100644 --- a/languages/python/python/stackencrypt/_declaration.py +++ b/languages/python/python/stackencrypt/_declaration.py @@ -8,10 +8,11 @@ The fields are the library's own list where one exists (dataclasses, attrs, Pydantic, ``NamedTuple``), and otherwise the class's type hints minus -``ClassVar`` and ``InitVar``, as on a SQLAlchemy declarative class. One generic -class around ``Annotated`` is read by structure, so ``Mapped[Annotated[str, -TextEq]]`` reads as ``TextEq``, with no SQLAlchemy import. A list, ``Final`` -and the other generics of the standard library are not read through. +``ClassVar``, ``InitVar`` and dunder names such as ``__tablename__``, as on a +SQLAlchemy declarative class. One generic class around ``Annotated`` is read +by structure, so ``Mapped[Annotated[str, TextEq]]`` reads as ``TextEq``, with +no SQLAlchemy import. A list, ``Final`` and the other generics of the +standard library are not read through. Then, in this order: @@ -80,6 +81,15 @@ class around ``Annotated`` is read by structure, so ``Mapped[Annotated[str, {"builtins", "collections", "collections.abc", "types", "typing"} ) +# The package whose wrappers, such as Mapped, are refused inside the marker's +# Annotated. A one-argument generic from any other package, such as +# re.Pattern[str], may be a value type. +_WRAPPER_PACKAGE = "sqlalchemy" + +# What evaluating a type hint raises when a name in it is missing or the +# string is not an expression. +_UNRESOLVED = (NameError, TypeError, AttributeError, SyntaxError) + # The one Python type of each marker kind. A value type must be exactly this # type: not Optional, not a union, and not a subclass. _KIND_TYPES: Dict[str, type] = { @@ -272,7 +282,7 @@ def _hints(cls: type) -> Dict[str, Any]: if not skip: try: return get_type_hints(cls, include_extras=True) - except (NameError, TypeError) as error: + except _UNRESOLVED as error: raise _hints_unresolved(cls, error) from error hints: Dict[str, Any] = {} # As get_type_hints does: each base's own hints in its own namespaces, @@ -280,24 +290,30 @@ def _hints(cls: type) -> Dict[str, Any]: for base in reversed(cls.__mro__): try: own = _own_annotations(base) - except (NameError, TypeError) as error: + except _UNRESOLVED as error: raise _hints_unresolved(cls, error) from error - module = sys.modules.get(base.__module__) - global_names = dict(vars(module)) if module is not None else {} for name, hint in own.items(): if name in skip: continue - - def probe() -> None: ... - - probe.__annotations__ = {name: hint} try: - hints.update(get_type_hints(probe, global_names, dict(vars(base)), include_extras=True)) - except (NameError, TypeError) as error: + hints[name] = _resolve_one(base, name, hint) + except _UNRESOLVED as error: raise _hints_unresolved(cls, error) from error return hints +def _resolve_one(base: type, name: str, hint: Any) -> Any: + """One hint of ``base``, resolved as ``get_type_hints`` resolves it. + + The probe is a class, not a function: a function's annotation cannot be + a ``ClassVar``, and before 3.14 a string ``ClassVar`` hint raises there. + """ + module = sys.modules.get(base.__module__) + global_names = dict(vars(module)) if module is not None else {} + probe = type("probe", (), {"__annotations__": {name: hint}}) + return get_type_hints(probe, global_names, dict(vars(base)), include_extras=True)[name] + + def _relationships(cls: type) -> frozenset: """The names of a SQLAlchemy class's relationships, such as ``books`` in ``books: Mapped[list["Book"]] = relationship()``. Empty for any other @@ -362,7 +378,9 @@ def _read_fields(cls: type, hints: Dict[str, Any]) -> List[_ReadField]: names = [ name for name, hint in hints.items() - if not _is_class_level(hint) and name not in relationships + if not _is_class_level(hint) + and not _is_dunder(name) + and name not in relationships ] read = [] for name in names: @@ -390,6 +408,12 @@ def _library_field_names(cls: type) -> Optional[List[str]]: return None +def _is_dunder(name: str) -> bool: + """A name such as ``__tablename__``: the class's configuration, which + the user cannot mark, not a field.""" + return len(name) > 4 and name.startswith("__") and name.endswith("__") + + def _is_class_level(hint: Any) -> bool: """A ``ClassVar`` or ``InitVar`` hint, which is not a field.""" if isinstance(hint, dataclasses.InitVar) or hint is dataclasses.InitVar: @@ -400,7 +424,17 @@ def _is_class_level(hint: Any) -> bool: def _hints_unresolved(cls: type, error: Exception) -> DeclarationError: - field = _unresolved_field(cls) + unresolved = _unresolved_field(cls) + field = unresolved[1] if unresolved else None + if unresolved is not None and _is_bare_class_var(*unresolved): + return DeclarationError( + "declaration_hints_unresolved", + f"field {field!r} is a bare ClassVar written as a string, which " + "this version of Python cannot read. Give it a type: " + "ClassVar[int], not ClassVar.", + field=field, + reason="bare_class_var", + ) where = f"field {field!r}" if field else f"a type hint of class {cls.__name__!r}" return DeclarationError( "declaration_hints_unresolved", @@ -411,8 +445,9 @@ def _hints_unresolved(cls: type, error: Exception) -> DeclarationError: ) -def _unresolved_field(cls: type) -> Optional[str]: - """The first field whose hint does not resolve, or ``None``. +def _unresolved_field(cls: type) -> Optional[Tuple[type, str, Any]]: + """The first field whose hint does not resolve, as the class that + declares it, its name and its hint, or ``None``. The hints are resolved one at a time, in the order ``get_type_hints`` resolves them, with the same namespaces. On 3.14, reading a class's @@ -421,22 +456,33 @@ def _unresolved_field(cls: type) -> Optional[str]: for base in reversed(cls.__mro__): try: own = _own_annotations(base) - except (NameError, TypeError): + except _UNRESOLVED: return None - module = sys.modules.get(base.__module__) - global_names = dict(vars(module)) if module is not None else {} for name, hint in own.items(): - - def probe() -> None: ... - - probe.__annotations__ = {name: hint} try: - get_type_hints(probe, global_names, dict(vars(base)), include_extras=True) - except (NameError, TypeError): - return name + _resolve_one(base, name, hint) + except _UNRESOLVED: + return base, name, hint return None +def _is_bare_class_var(base: type, name: str, hint: Any) -> bool: + """Whether ``hint`` is a string that names ``ClassVar`` with no type. + + Python 3.9 cannot read one, as the future import writes it. The string is + evaluated as ``get_type_hints`` evaluates it, so ``typing.ClassVar`` or an + alias of it is found too. + """ + if not isinstance(hint, str): + return False + module = sys.modules.get(base.__module__) + global_names = dict(vars(module)) if module is not None else {} + try: + return eval(hint, global_names, dict(vars(base))) is ClassVar # noqa: S307 + except Exception: # noqa: BLE001 Any other failure is not this case. + return False + + def _own_annotations(base: type) -> Dict[str, Any]: """The annotations written on ``base`` itself, not on its bases.""" if sys.version_info < (3, 10): @@ -478,6 +524,14 @@ def _is_wrapper(hint: Any) -> bool: ) +def _is_orm_wrapper(hint: Any) -> bool: + """A SQLAlchemy wrapper, such as ``Mapped[str]``, found by its module.""" + return ( + _is_wrapper(hint) + and get_origin(hint).__module__.partition(".")[0] == _WRAPPER_PACKAGE + ) + + def _without_none(value_type: Any) -> Tuple[Any, bool]: """``Optional[T]`` as ``(T, True)``; any other type as ``(type, False)``. ``Optional[T]`` is ``Union[T, None]``, and ``T | None`` from Python 3.10. @@ -494,9 +548,10 @@ def _marker_of(name: str, hint: Any, *, from_hints: bool) -> Tuple[Type[Marker], """The one marker of a field and its value type, or a ``DeclarationError``. ``from_hints`` says the field list came from the type hints, as on a - SQLAlchemy class. Only there is a wrapper inside the marker's + SQLAlchemy class. Only there is a SQLAlchemy wrapper inside the marker's ``Annotated`` refused as misplaced: ``Annotated[Mapped[str], TextEq]``. - Elsewhere a generic value type is a type mismatch, checked later. + Any other generic value type, such as ``re.Pattern[str]``, is the value's + own: a sealed marker refuses it later, as a type mismatch. """ if hint is None: raise _no_marker(name) @@ -512,7 +567,7 @@ def _marker_of(name: str, hint: Any, *, from_hints: bool) -> Tuple[Type[Marker], # Annotated outermost, with the wrapper inside it, is the order reversed. # Inside a wrapper already read, a generic value type is the value's own, # as in a relationship: Mapped[Annotated[list[Post], Ignore]]. - if from_hints and not unwrapped and markers and _is_wrapper(value_type): + if from_hints and not unwrapped and markers and _is_orm_wrapper(value_type): raise _marker_not_outermost(name, markers[0], value_type) if _nested_markers(hint): raise _marker_not_outermost(name, None, None) @@ -627,7 +682,8 @@ def _check_constructor(cls: type, read: List[_ReadField]) -> None: for field in read: parameter = parameters.get(field.name) if field.marker is Ignore: - if parameter is not None and _required(parameter): + taken_as = parameters.get(_init_name(cls, field.name)) + if taken_as is not None and _required(taken_as): raise DeclarationError( "declaration_ignore_required", f"field {field.name!r} is marked Ignore, but the constructor " @@ -660,20 +716,46 @@ def _takes( cls: type, name: str, parameter: Optional[inspect.Parameter], any_keyword: bool ) -> bool: """Whether the constructor takes field ``name`` as a keyword argument.""" - fields = getattr(cls, "__pydantic_fields__", None) - info = fields.get(name) if isinstance(fields, dict) else None + info = _pydantic_field(cls, name) if info is not None: # Pydantic's signature shows a field's alias, not its name. Its config # says whether the name is taken too. With extra="allow" the signature - # has **kwargs, and a name that is not taken lands in the extras. + # has **kwargs, and a name that is not taken lands in the extras. A + # Pydantic dataclass keeps its config in __pydantic_config__. alias = getattr(info, "validation_alias", None) or getattr(info, "alias", None) if alias is None or alias == name: return True - config = getattr(cls, "model_config", {}) + config = ( + getattr(cls, "model_config", None) or getattr(cls, "__pydantic_config__", None) or {} + ) return bool(config.get("populate_by_name") or config.get("validate_by_name")) return _keyword(parameter) or any_keyword +def _pydantic_field(cls: type, name: str) -> Any: + """Pydantic's record of field ``name``, or ``None``.""" + fields = getattr(cls, "__pydantic_fields__", None) + return fields.get(name) if isinstance(fields, dict) else None + + +def _init_name(cls: type, name: str) -> str: + """The parameter name the constructor gives field ``name``. + + Pydantic names it by the field's alias. attrs names a private field + without its leading underscore, so ``_cache`` is ``cache``. + """ + info = _pydantic_field(cls, name) + if info is not None: + for alias in (getattr(info, "validation_alias", None), getattr(info, "alias", None)): + if isinstance(alias, str): + return alias + return name + for attribute in getattr(cls, "__attrs_attrs__", ()): + if attribute.name == name: + return getattr(attribute, "alias", None) or name + return name + + def _keyword(parameter: Optional[inspect.Parameter]) -> bool: return parameter is not None and parameter.kind in ( inspect.Parameter.POSITIONAL_OR_KEYWORD, diff --git a/languages/python/python/stackencrypt/_redact.py b/languages/python/python/stackencrypt/_redact.py index 2c60ad0bb..39d1eff96 100644 --- a/languages/python/python/stackencrypt/_redact.py +++ b/languages/python/python/stackencrypt/_redact.py @@ -7,17 +7,17 @@ Pydantic's ``model_dump``, ``vars`` and ``__dict__`` still return plaintext, and so does a debugger that reads attributes. -A generated method (dataclass, attrs, Pydantic, ``NamedTuple``) is replaced. -A method written on the class itself is refused unless the decorator is +A method that dataclasses, attrs or ``NamedTuple`` generated is replaced. +Pydantic's methods are on its base classes. Anything else on the class +itself is the user's: a function, a ``staticmethod``, a ``partialmethod``, +or a function from another module. It is refused unless the decorator is given ``redact=False``, because replacing it would silently drop the user's code. A method inherited from a base is replaced on the declared class only. """ import collections -import dataclasses import inspect import os -import reprlib import types from typing import Any, Callable, Iterator, Sequence, Tuple @@ -45,38 +45,53 @@ def __repr__(self) -> str: type(vars(collections.namedtuple("_Probe", "field"))["field"]), ) -# The files a generated __repr__ reports as its own. A dataclass's __repr__ is -# wrapped for recursion, and the wrapper is in reprlib.py on 3.14 and in -# dataclasses.py on 3.9. attrs reports a "" name. -_GENERATOR_FILES = frozenset( - os.path.realpath(module.__file__) - for module in (dataclasses, reprlib) - if module.__file__ is not None -) - # attrs compiles its methods under this file name. _ATTRS_GENERATED = " bool: - """Whether ``cls`` itself has a ``name`` method written by hand. + """Whether ``cls`` itself has a ``name`` that no known generator wrote. Only the class's own ``__dict__`` is read: a base's method is not the - class's. A generated method comes from a generator's file, or from - another module, such as ``pydantic.main``. A method compiled from a - string, as in the REPL, a notebook or ``python -c``, is hand-written. + class's. Anything there is the user's unless it is a generator's output. + A method compiled from a string, as in the REPL, a notebook or + ``python -c``, is hand-written. + """ + found = vars(cls) + return name in found and not _generated(found[name]) + + +def _generated(method: object) -> bool: + """Whether dataclasses, attrs or ``NamedTuple`` generated ``method``. + + A wrapper, such as the one ``reprlib.recursive_repr`` adds, is + unwrapped first, so the function inside it decides. A user's + ``recursive_repr`` is then the user's own function. """ - method = vars(cls).get(name) - code = getattr(method, "__code__", None) - if code is None or getattr(method, "__module__", None) != cls.__module__: + if not isinstance(method, types.FunctionType): + return False + inner = inspect.unwrap(method) + code = getattr(inner, "__code__", None) + if code is None: return False filename = code.co_filename - return ( - not filename.startswith(_ATTRS_GENERATED) - and os.path.realpath(filename) not in _GENERATOR_FILES - ) + if filename.startswith(_ATTRS_GENERATED): + return True + if filename == _DATACLASS_FILE: + return inner.__qualname__.startswith(_DATACLASS_QUALNAME) + return _COLLECTIONS_FILE is not None and os.path.realpath(filename) == _COLLECTIONS_FILE def install(cls: type, fields: Sequence[Tuple[str, bool]]) -> Tuple[Callable[..., Any], ...]: @@ -105,6 +120,16 @@ def __rich_repr__(self: object) -> Iterator[Tuple[str, object]]: for method in methods: method.__qualname__ = f"{cls.__qualname__}.{method.__name__}" setattr(cls, method.__name__, method) + # Pydantic builds its other print hooks from __repr_args__: __repr_str__, + # and __pretty__, which devtools' debug() prints through. It is replaced + # too, with the same fields. Cipher does not check it again. + if isinstance(getattr(cls, "__pydantic_fields__", None), dict): + + def __repr_args__(self: object) -> Iterator[Tuple[str, object]]: + return __rich_repr__(self) + + __repr_args__.__qualname__ = f"{cls.__qualname__}.__repr_args__" + setattr(cls, "__repr_args__", __repr_args__) return methods diff --git a/languages/python/tests/_compiled_cases.py b/languages/python/tests/_compiled_cases.py new file mode 100644 index 000000000..f0f2714ad --- /dev/null +++ b/languages/python/tests/_compiled_cases.py @@ -0,0 +1,25 @@ +"""Compile ``_declaration_cases.py`` with pytest's assertion rewriting. + +The cases run through ``exec``, so pytest does not rewrite their ``assert`` +statements as it rewrites a test module's. A failure would then show a bare +``AssertionError`` with no values. This applies the same rewrite to the +parsed source first. ``rewrite_asserts`` is in pytest's private ``_pytest`` +package, so a pytest release can change it; a change fails both declaration +test modules at import. +""" + +import ast +import pathlib +from types import CodeType + +from _pytest.assertion.rewrite import rewrite_asserts + +CASES = pathlib.Path(__file__).with_name("_declaration_cases.py") + + +def compiled_cases(flags: int = 0) -> CodeType: + """The cases, with their asserts rewritten, compiled with ``flags``.""" + source = CASES.read_bytes() + tree = ast.parse(source, filename=str(CASES)) + rewrite_asserts(tree, source, str(CASES)) + return compile(tree, str(CASES), "exec", flags=flags, dont_inherit=True) diff --git a/languages/python/tests/_declaration_cases.py b/languages/python/tests/_declaration_cases.py index 5992001e7..44f8a0b87 100644 --- a/languages/python/tests/_declaration_cases.py +++ b/languages/python/tests/_declaration_cases.py @@ -10,12 +10,15 @@ import datetime import decimal import pickle +import re import sys +import typing import uuid from typing import Annotated, ClassVar, Final, List, NamedTuple, Optional, Union import attrs import pydantic +import pydantic.dataclasses import pytest from sqlalchemy import inspect from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship @@ -236,6 +239,29 @@ class User(KeywordConstructor): assert planned(User) == [("email", TextEq)] +def test_a_generic_value_type_from_another_module_is_not_a_misplaced_wrapper(): + # re.Pattern[str] is a one-argument generic, but it is a value type, not + # a wrapper such as Mapped. + @stackencrypt.model("users") + class User(KeywordConstructor): + email: Annotated[str, TextEq] + pattern: Annotated[re.Pattern[str], Ignore] + + assert planned(User) == [("email", TextEq)] + + +def test_a_dunder_annotation_is_not_a_field(): + # As on a SQLAlchemy class: __tablename__ is configuration, not a column, + # and the user cannot mark it. + @stackencrypt.model("users") + class User(KeywordConstructor): + __tablename__: str = "users" + __table_args__: tuple = () + email: Annotated[str, TextEq] + + assert planned(User) == [("email", TextEq)] + + @pytest.mark.parametrize("kind", ["dataclass", "plain"]) def test_a_marker_inside_a_list_or_final_is_refused(kind): def build_list(): @@ -305,6 +331,66 @@ class User: assert isinstance(error.__cause__, NameError) +def build_with_a_bad_hint(kind): + if kind == "missing attribute": + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + posts: Annotated[List["sys.Undefined"], Ignore] = dataclasses.field( + default_factory=list + ) + + else: + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + posts: "List[" = dataclasses.field(default_factory=list) # noqa: F722 + + +@pytest.mark.parametrize( + "kind, cause", + [("missing attribute", AttributeError), ("malformed string", SyntaxError)], +) +def test_a_hint_that_fails_to_evaluate_is_refused_with_its_field(kind, cause): + error = refusal(lambda: build_with_a_bad_hint(kind)) + assert error.code == "declaration_hints_unresolved" + assert error.field == "posts" + assert isinstance(error.__cause__, cause) + + +def build_with_a_class_var_before_an_unresolved_hint(kind): + # A ClassVar hint written as a string resolves on a class, not as a + # function argument. Pydantic's own bases hold such hints. + if kind == "dataclass": + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + count: "ClassVar[int]" = 0 + email: Annotated[str, TextEq] = "" + posts: Annotated[List["Undefined"], Ignore] = dataclasses.field( # noqa: F821 + default_factory=list + ) + + else: + + @stackencrypt.model("users") + class User(pydantic.BaseModel): + email: Annotated[str, TextEq] + posts: Annotated[List["Undefined"], Ignore] = [] # noqa: F821 + + +@pytest.mark.parametrize("kind", ["dataclass", "pydantic"]) +def test_an_unresolved_hint_after_a_class_var_is_named(kind): + error = refusal(lambda: build_with_a_class_var_before_an_unresolved_hint(kind)) + assert error.code == "declaration_hints_unresolved" + assert error.field == "posts" + + class LaterBase(DeclarativeBase): pass @@ -395,6 +481,73 @@ class User: assert planned(User) == [("email", TextEq)] +def test_class_level_hints_are_not_fields_on_a_class_read_from_its_hints(): + # No library field list, so the fields come from the hints, as on a + # SQLAlchemy class. Each form the filter checks is here. + @stackencrypt.model("users") + class User(KeywordConstructor): + email: Annotated[str, TextEq] + registry: ClassVar[dict] = {} + token: dataclasses.InitVar[str] = "" + if sys.version_info >= (3, 10): + # Python 3.9 cannot resolve a bare ClassVar written as a string, + # as the future import writes it: get_type_hints raises. + bare: ClassVar = 0 + + assert planned(User) == [("email", TextEq)] + + +def test_a_bare_class_var_python_cannot_read_is_refused_with_advice(): + # Python 3.9 cannot read a bare ClassVar written as a string, as the + # future import writes it. Python 3.14 can. The test asks this Python. + class User(KeywordConstructor): + email: Annotated[str, TextEq] + count: ClassVar = 0 + + try: + typing.get_type_hints(User, include_extras=True) + except TypeError: + error = refusal(lambda: stackencrypt.model("users")(User)) + assert error.code == "declaration_hints_unresolved" + assert error.field == "count" + assert error.reason == "bare_class_var" + assert "ClassVar[int]" in str(error) + else: + assert planned(stackencrypt.model("users")(User)) == [("email", TextEq)] + + +class RefusedBase(DeclarativeBase): + pass + + +def build_a_mapped_class(kind): + if kind == "no marker": + + @stackencrypt.model("mapped_no_marker") + class MappedNoMarker(RefusedBase): + __tablename__ = "mapped_no_marker" + id: Mapped[Annotated[int, Passthrough]] = mapped_column(primary_key=True) + email: Mapped[str] + + else: + + @stackencrypt.model("mapped_two_markers") + class MappedTwoMarkers(RefusedBase): + __tablename__ = "mapped_two_markers" + id: Mapped[Annotated[int, Passthrough]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq, Passthrough]] + + +@pytest.mark.parametrize( + "kind, code", + [("no marker", "declaration_no_marker"), ("two markers", "declaration_two_markers")], +) +def test_rule_a_holds_inside_mapped(kind, code): + error = refusal(lambda: build_a_mapped_class(kind)) + assert error.code == code + assert error.field == "email" + + def test_a_subclass_is_not_declared_until_it_is_decorated(): @stackencrypt.model("users") @dataclasses.dataclass @@ -615,6 +768,13 @@ class User: class User: _email: Annotated[str, TextEq] + elif kind == "pydantic dataclass init=False field": + + @stackencrypt.model("users") + @pydantic.dataclasses.dataclass + class User: + email: Annotated[str, TextEq] = dataclasses.field(init=False) + elif kind == "pydantic alias": @stackencrypt.model("users") @@ -635,6 +795,7 @@ class User(pydantic.BaseModel): "init=False field", "init=False class", "attrs private name", + "pydantic dataclass init=False field", "pydantic alias", "pydantic alias with extra=allow", ], @@ -654,6 +815,16 @@ class User(pydantic.BaseModel): assert planned(User) == [("email", TextEq)] +def test_a_pydantic_dataclass_alias_is_accepted_when_its_config_takes_the_name(): + # A Pydantic dataclass keeps its config in __pydantic_config__. + @stackencrypt.model("users") + @pydantic.dataclasses.dataclass(config=pydantic.ConfigDict(populate_by_name=True)) + class User: + email: Annotated[str, TextEq] = pydantic.Field(alias="Email") + + assert planned(User) == [("email", TextEq)] + + def build_with_a_required_ignore(kind): if kind == "dataclass": @@ -671,6 +842,21 @@ class User: email: Annotated[str, TextEq] note: Annotated[str, Ignore] + elif kind == "attrs private name": + + @stackencrypt.model("users") + @attrs.define + class User: + email: Annotated[str, TextEq] + _note: Annotated[str, Ignore] + + elif kind == "pydantic alias": + + @stackencrypt.model("users") + class User(pydantic.BaseModel): + email: Annotated[str, TextEq] + note: Annotated[str, Ignore] = pydantic.Field(alias="Note") + else: @stackencrypt.model("users") @@ -679,11 +865,20 @@ class User(pydantic.BaseModel): note: Annotated[str, Ignore] -@pytest.mark.parametrize("kind", ["dataclass", "attrs", "pydantic"]) -def test_a_required_ignore_field_is_refused(kind): +@pytest.mark.parametrize( + "kind, field", + [ + ("dataclass", "note"), + ("attrs", "note"), + ("attrs private name", "_note"), + ("pydantic", "note"), + ("pydantic alias", "note"), + ], +) +def test_a_required_ignore_field_is_refused(kind, field): error = refusal(lambda: build_with_a_required_ignore(kind)) assert error.code == "declaration_ignore_required" - assert error.field == "note" + assert error.field == field assert "default" in str(error) diff --git a/languages/python/tests/_fork.py b/languages/python/tests/_fork.py new file mode 100644 index 000000000..0720c06e2 --- /dev/null +++ b/languages/python/tests/_fork.py @@ -0,0 +1,56 @@ +"""Child process of `test_concurrency_live.py`: a fork, and a call in the +forked child. Prints what the child did as JSON. + +`after` forks after the first call, so the parent's client exists: the +child must be refused, with no wait. `before` forks before any call, so no +client exists: the child must build its own and work. + +The child's result comes back through a pipe. The parent waits at most 30 +seconds, so a child that never returns shows as "hang" and does not stop +the test run. +""" + +import json +import os +import sys +import time + +from _sync_contract import User, user + +from stackencrypt import Cipher, StashError, _engine + +mode = sys.argv[1] +cipher = Cipher(User) +if mode == "after": + cipher.encrypt([user(1)]) + assert _engine.client_builds() == 1 + +read_end, write_end = os.pipe() +pid = os.fork() +if pid == 0: + os.close(read_end) + try: + rows = cipher.encrypt([user(2)]) + opened = cipher.decrypt(rows) + result = {"child": "ok" if opened == [user(2)] else "wrong"} + except StashError as error: + result = {"child": error.code} + except BaseException as error: # noqa: BLE001 Reported to the parent. + result = {"child": f"error: {type(error).__name__}"} + os.write(write_end, json.dumps(result).encode()) + os._exit(0) + +os.close(write_end) +deadline = time.monotonic() + 30 +while time.monotonic() < deadline: + done, status = os.waitpid(pid, os.WNOHANG) + if done: + break + time.sleep(0.05) +else: + os.kill(pid, 9) + os.waitpid(pid, 0) + print(json.dumps({"child": "hang"})) + raise SystemExit(0) + +print(os.read(read_end, 4096).decode() or json.dumps({"child": f"no result, status {status}"})) diff --git a/languages/python/tests/_live.py b/languages/python/tests/_live.py index cd1726ab1..a58dcac93 100644 --- a/languages/python/tests/_live.py +++ b/languages/python/tests/_live.py @@ -62,4 +62,6 @@ def run_python(args: List[str], env: Optional[Dict[str, str]] = None) -> Any: timeout=120, ) assert done.returncode == 0, done.stderr - return json.loads(done.stdout.strip().splitlines()[-1]) + lines = done.stdout.strip().splitlines() + assert lines, f"the child printed nothing. Its stderr:\n{done.stderr}" + return json.loads(lines[-1]) diff --git a/languages/python/tests/_readme.py b/languages/python/tests/_readme.py index 1764a428c..02ac1e230 100644 --- a/languages/python/tests/_readme.py +++ b/languages/python/tests/_readme.py @@ -53,27 +53,41 @@ def program() -> str: class _TopLevel(ast.NodeVisitor): - """The names a program binds at its top level, and the names it reads - there. A function, class, lambda, or comprehension has a scope of its - own, so its names are skipped.""" + """The names a program binds at its top level. A function, class, + lambda, or comprehension has a scope of its own, so its names are + skipped.""" def __init__(self) -> None: self.bound: Set[str] = set() - self.read: Set[str] = set() def visit_Name(self, node: ast.Name) -> None: - (self.bound if isinstance(node.ctx, ast.Store) else self.read).add(node.id) + if isinstance(node.ctx, ast.Store): + self.bound.add(node.id) def visit_alias(self, node: ast.alias) -> None: self.bound.add((node.asname or node.name).split(".")[0]) + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None: + # `except E as name` binds `name`, which is a str, not an ast.Name. + if node.name: + self.bound.add(node.name) + self.generic_visit(node) + def _own_scope(self, node: ast.AST) -> None: if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): self.bound.add(node.name) + def _comprehension(self, node: ast.AST) -> None: + # A comprehension's own variables stay inside it, but a walrus + # (`:=`) in it binds its target in the enclosing scope. + for inner in ast.walk(node): + if isinstance(inner, ast.NamedExpr) and isinstance(inner.target, ast.Name): + self.bound.add(inner.target.id) + visit_FunctionDef = visit_AsyncFunctionDef = visit_ClassDef = _own_scope - visit_Lambda = visit_ListComp = visit_SetComp = visit_DictComp = _own_scope - visit_GeneratorExp = _own_scope + visit_Lambda = _own_scope + visit_ListComp = visit_SetComp = visit_DictComp = _comprehension + visit_GeneratorExp = _comprehension def _top_level(source: str) -> _TopLevel: diff --git a/languages/python/tests/_sync_contract.py b/languages/python/tests/_sync_contract.py index 3fa6e9914..6d3564d75 100644 --- a/languages/python/tests/_sync_contract.py +++ b/languages/python/tests/_sync_contract.py @@ -13,6 +13,7 @@ import signal import threading import time +import traceback from typing import Annotated, Dict, List import stackencrypt @@ -39,15 +40,16 @@ def encrypt_from_threads(cipher: Cipher, per_thread: int) -> List[Dict[str, obje any call failed.""" sealed: List[List[Dict[str, object]]] = [[] for _ in range(THREADS)] failures: List[str] = [] - start = threading.Barrier(THREADS) + # A timeout, so one thread that never arrives fails the test, not hangs it. + start = threading.Barrier(THREADS, timeout=30) def encrypt_many(t: int) -> None: - start.wait() try: + start.wait() for n in range(per_thread): sealed[t].extend(cipher.encrypt([user(t * per_thread + n)])) - except Exception as error: - failures.append(repr(error)) + except Exception: + failures.append(f"thread {t}: {traceback.format_exc()}") threads = [threading.Thread(target=encrypt_many, args=(t,)) for t in range(THREADS)] for thread in threads: @@ -58,6 +60,32 @@ def encrypt_many(t: int) -> None: return [row for rows in sealed for row in rows] +def decrypt_from_threads(cipher: Cipher, rows: List[Dict[str, object]]) -> List[User]: + """Release ``THREADS`` threads at once, each to decrypt its own slice of + ``rows`` one call at a time. Returns every user, in row order. Raises if + any call failed.""" + per_thread = -(-len(rows) // THREADS) + opened: List[List[User]] = [[] for _ in range(THREADS)] + failures: List[str] = [] + start = threading.Barrier(THREADS, timeout=30) + + def decrypt_many(t: int) -> None: + try: + start.wait() + for row in rows[t * per_thread : (t + 1) * per_thread]: + opened[t].extend(cipher.decrypt([row])) + except Exception: + failures.append(f"thread {t}: {traceback.format_exc()}") + + threads = [threading.Thread(target=decrypt_many, args=(t,)) for t in range(THREADS)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert failures == [], failures + return [model for models in opened for model in models] + + def send_sigint_later(seconds: float) -> threading.Timer: """Send SIGINT to this process after ``seconds``.""" timer = threading.Timer(seconds, os.kill, (os.getpid(), signal.SIGINT)) @@ -70,4 +98,6 @@ def wait_for_the_interrupt() -> None: loop gives it that check, and ends if it never comes.""" deadline = time.monotonic() + 5 while time.monotonic() < deadline: - pass + # A short sleep still returns to Python, which then checks for the + # signal, and it does not keep a core busy. + time.sleep(0.01) diff --git a/languages/python/tests/conftest.py b/languages/python/tests/conftest.py index 21f0c53bc..70a4e936d 100644 --- a/languages/python/tests/conftest.py +++ b/languages/python/tests/conftest.py @@ -36,7 +36,18 @@ def no_credentials(monkeypatch, tmp_path): def credentials(monkeypatch, no_credentials): """Credentials that parse, so ``Cipher`` builds, and no client key. A call that gets past every check fails building the engine client, with - no request.""" + no request. + + The engine client is shared by the whole process. Once a live test has + built it, a call here would reach ZeroKMS instead of failing, so the + test is skipped. ``mise run test`` and ``-m live`` run the two kinds of + test in separate processes, and skip nothing. + """ + if _engine.client_builds() > 0: + pytest.skip( + "a live test built the shared engine client earlier in this " + "process; run the offline tests in a process of their own" + ) monkeypatch.setenv("CS_CLIENT_ACCESS_KEY", ACCESS_KEY) monkeypatch.setenv("CS_WORKSPACE_CRN", WORKSPACE_CRN) diff --git a/languages/python/tests/test_cipher.py b/languages/python/tests/test_cipher.py index 7876481a5..a3b619136 100644 --- a/languages/python/tests/test_cipher.py +++ b/languages/python/tests/test_cipher.py @@ -128,6 +128,44 @@ def test_a_replaced_repr_is_refused(credentials): assert "__repr__ replaced" in str(error) +@pytest.mark.parametrize("change", ["changed", "removed"]) +def test_a_changed_or_removed_hint_is_refused(credentials, change): + User = declared() + if change == "changed": + User.__annotations__["age"] = Annotated[str, Passthrough] + else: + del User.__annotations__["age"] + + error = refusal(lambda: Cipher(User)) + assert error.code == "declaration_class_changed" + assert error.reason == "fields" + assert error.field == "age" + + +@pytest.mark.parametrize("name", ["__str__", "__rich_repr__"]) +def test_each_replaced_method_is_refused(credentials, name): + User = declared() + setattr(User, name, lambda self: self.email) + + error = refusal(lambda: Cipher(User)) + assert error.code == "declaration_class_changed" + assert error.reason == "methods" + assert f"{name} replaced" in str(error) + + +def test_a_redact_false_class_may_replace_its_printing_methods(credentials): + # The README says Cipher checks the printing methods only when the + # decorator replaced them. + @stackencrypt.model("users", redact=False) + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + + User.__repr__ = lambda self: "replaced" + + Cipher(User) + + def test_a_hint_that_no_longer_resolves_names_the_name(credentials): User = declared() User.__annotations__["posts"] = "List[Undefined]" diff --git a/languages/python/tests/test_concurrency_live.py b/languages/python/tests/test_concurrency_live.py index 3d876c612..1fcb16fb6 100644 --- a/languages/python/tests/test_concurrency_live.py +++ b/languages/python/tests/test_concurrency_live.py @@ -13,7 +13,14 @@ import pytest from _live import LIVE, run_python -from _sync_contract import PER_THREAD, THREADS, User, encrypt_from_threads, user +from _sync_contract import ( + PER_THREAD, + THREADS, + User, + decrypt_from_threads, + encrypt_from_threads, + user, +) from stackencrypt import Cipher @@ -34,6 +41,30 @@ def test_threads_encrypt_at_once_and_every_row_opens(): assert cipher.decrypt(rows) == [user(n) for n in range(THREADS * PER_THREAD)] +def test_threads_decrypt_at_once_and_every_row_opens(): + cipher = Cipher(User) + rows = encrypt_from_threads(cipher, PER_THREAD) + + assert decrypt_from_threads(cipher, rows) == [user(n) for n in range(THREADS * PER_THREAD)] + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows has no fork") +@pytest.mark.parametrize( + "mode, child", + [ + # The parent's client cannot work in the child, and a new runtime + # there can block. The child is refused at once, with what to do. + ("after", "stack_forked"), + # No client existed at the fork, so the child builds its own. + ("before", "ok"), + ], +) +def test_a_forked_child_is_refused_only_when_the_client_was_built_before_the_fork( + mode, child +): + assert run_python([str(HERE / "_fork.py"), mode]) == {"child": child} + + @pytest.mark.skipif(sys.platform == "win32", reason="SIGINT is sent with os.kill") def test_an_interrupt_during_encrypt_waits_for_it_and_the_cipher_still_works(): result = run_python([str(HERE / "_interrupt.py")]) diff --git a/languages/python/tests/test_declaration.py b/languages/python/tests/test_declaration.py index 4ef9304f1..3506cb3ce 100644 --- a/languages/python/tests/test_declaration.py +++ b/languages/python/tests/test_declaration.py @@ -1,10 +1,10 @@ """The declaration tests, with annotations evaluated as written. The tests themselves are in ``_declaration_cases.py``, compiled here without -``from __future__ import annotations``. +``from __future__ import annotations``, with their asserts rewritten as +pytest rewrites a test module's (``_compiled_cases.py``). """ -import pathlib +from _compiled_cases import compiled_cases -_CASES = pathlib.Path(__file__).with_name("_declaration_cases.py") -exec(compile(_CASES.read_text(), str(_CASES), "exec", dont_inherit=True), globals()) +exec(compiled_cases(), globals()) diff --git a/languages/python/tests/test_declaration_future_annotations.py b/languages/python/tests/test_declaration_future_annotations.py index 3d9d4f5a7..57dd04403 100644 --- a/languages/python/tests/test_declaration_future_annotations.py +++ b/languages/python/tests/test_declaration_future_annotations.py @@ -1,20 +1,12 @@ """The declaration tests, compiled with ``from __future__ import annotations``. Under that import every annotation is a string until the decorator resolves -it, which is the case a real model module with the import is in. +it, which is the case a real model module with the import is in. The asserts +are rewritten as pytest rewrites a test module's (``_compiled_cases.py``). """ import __future__ -import pathlib -_CASES = pathlib.Path(__file__).with_name("_declaration_cases.py") -exec( - compile( - _CASES.read_text(), - str(_CASES), - "exec", - flags=__future__.annotations.compiler_flag, - dont_inherit=True, - ), - globals(), -) +from _compiled_cases import compiled_cases + +exec(compiled_cases(__future__.annotations.compiler_flag), globals()) diff --git a/languages/python/tests/test_decrypt.py b/languages/python/tests/test_decrypt.py index 9a4c94768..2bba9b707 100644 --- a/languages/python/tests/test_decrypt.py +++ b/languages/python/tests/test_decrypt.py @@ -7,7 +7,9 @@ """ import dataclasses +import datetime import json +import uuid from typing import Annotated import pytest @@ -57,6 +59,79 @@ def test_a_tuple_row_is_the_wrong_shape_and_names_dict_row(untouched): assert "dict_row" in error.message +def test_one_row_on_its_own_is_refused_as_one_row(untouched): + # As from cursor.fetchone(): list() of a dict is its column names. + error = refused({"email": STORED, "age": 3}) + assert error.code == "decrypt_row_shape" + assert "not one row" in error.message + + +def test_a_dict_that_is_not_json_is_malformed(untouched): + # A set has no JSON form: json.dumps raises TypeError. + value = {"i": {"t": "users", "c": "email"}, "v": {1, 2}} + error = refused([{"email": value, "age": 3}]) + assert error.code == "decrypt_malformed" + assert error.field == "email" + assert "row 0" in error.message + + +def test_a_str_that_is_not_valid_utf8_is_malformed(untouched): + # A lone surrogate: a str Python holds, with no UTF-8 encoding. + error = refused([{"email": "\ud800", "age": 3}]) + assert error.code == "decrypt_malformed" + assert error.field == "email" + + +class Boom(Exception): + """An error that is not about the input's JSON or type.""" + + +class ExplodingDict(dict): + """A dict whose items, which json.dumps reads, raise Boom.""" + + def items(self): + raise Boom("not about the JSON") + + +def test_an_error_that_is_not_about_the_json_propagates(untouched): + with pytest.raises(Boom): + Cipher(User).decrypt([{"email": ExplodingDict(i={}), "age": 3}]) + + +class NoOffset(datetime.tzinfo): + def utcoffset(self, dt): + return None + + +class RaisingOffset(datetime.tzinfo): + def utcoffset(self, dt): + raise Boom("the user's own tzinfo") + + +@stackencrypt.model("visits") +@dataclasses.dataclass +class Visit: + email: Annotated[str, TextEq] + seen: Annotated[datetime.datetime, Passthrough] + + +VISIT_STORED = json.dumps({"i": {"t": "visits", "c": "email"}, "v": 2}) + + +def test_a_passthrough_datetime_whose_tzinfo_gives_no_offset_is_naive(untouched): + seen = datetime.datetime(2026, 1, 1, tzinfo=NoOffset()) + with pytest.raises(InvalidValueError) as raised: + Cipher(Visit).decrypt([{"email": VISIT_STORED, "seen": seen}]) + assert raised.value.code == "value_naive_datetime" + assert raised.value.field == "seen" + + +def test_an_error_from_a_passthrough_value_propagates(untouched): + seen = datetime.datetime(2026, 1, 1, tzinfo=RaisingOffset()) + with pytest.raises(Boom): + Cipher(Visit).decrypt([{"email": VISIT_STORED, "seen": seen}]) + + def test_a_row_without_a_declared_column_names_it(untouched): error = refused([{"email": STORED}]) assert error.code == "decrypt_missing_column" @@ -115,6 +190,23 @@ def test_a_passthrough_int_that_arrives_as_str_is_the_wrong_type(untouched): assert error.reason == "passthrough" +@stackencrypt.model("keyed") +@dataclasses.dataclass +class Keyed: + email: Annotated[str, TextEq] + id: Annotated[str, Passthrough] + + +def test_a_uuid_for_a_str_passthrough_is_the_wrong_type(untouched): + # The README's UUID advice depends on this: psycopg reads a uuid column + # as a uuid.UUID, and decrypt does not convert it to str. + stored = json.dumps({"i": {"t": "keyed", "c": "email"}, "v": 2}) + with pytest.raises(InvalidValueError) as raised: + Cipher(Keyed).decrypt([{"email": stored, "id": uuid.uuid4()}]) + assert raised.value.code == "value_wrong_type" + assert raised.value.field == "id" + + def test_a_null_passthrough_is_refused(untouched): error = refused([{"email": STORED, "age": None}], InvalidValueError) assert error.code == "value_none" diff --git a/languages/python/tests/test_decrypt_rows.py b/languages/python/tests/test_decrypt_rows.py index be545bcb3..170f43d90 100644 --- a/languages/python/tests/test_decrypt_rows.py +++ b/languages/python/tests/test_decrypt_rows.py @@ -58,6 +58,12 @@ def test_rows_with_no_declared_field_come_back_unchanged_with_no_request(people, assert people.decrypt_rows(rows) == rows +def test_one_row_on_its_own_is_refused_as_one_row(people, untouched): + error = refused(people, {"email": STORED, "age": 3}) + assert error.code == "decrypt_row_shape" + assert "not one row" in error.message + + def test_a_row_that_is_not_a_mapping_is_refused(people, untouched): error = refused(people, [{"email": STORED}, (STORED,)]) assert error.code == "decrypt_row_shape" diff --git a/languages/python/tests/test_encrypt.py b/languages/python/tests/test_encrypt.py index 305a92445..2978559d8 100644 --- a/languages/python/tests/test_encrypt.py +++ b/languages/python/tests/test_encrypt.py @@ -44,10 +44,17 @@ def cipher_for(**fields): def refused(cipher, model): + """The refusal of ``model``. Neither the valid email nor the refused + value may appear in it. None and a bool are not checked: their text can + appear in a message for another reason.""" with pytest.raises(InvalidValueError) as raised: cipher.encrypt([model]) error = raised.value assert SECRET not in str(error) + value = getattr(model, error.field) if error.field else None + if value is not None and not isinstance(value, bool): + for shown in {repr(value), str(value)}: + assert shown not in str(error), f"{shown} is in {error}" return error @@ -102,7 +109,7 @@ def test_a_passthrough_value_of_another_type_is_refused(untouched): def test_an_int_outside_its_kind_is_out_of_range(untouched): cipher, User = cipher_for(rank=Annotated[int, Smallint]) - error = refused(cipher, User(email=SECRET, rank=40_000)) + error = refused(cipher, User(email=SECRET, rank=40_123)) assert error.code == "value_out_of_range" assert error.field == "rank" assert error.reason == "int16" @@ -149,6 +156,29 @@ def test_a_naive_datetime_is_refused(untouched): assert "tzinfo" in error.message +class NoOffset(datetime.tzinfo): + """A tzinfo that gives no offset: Python counts its datetime as naive.""" + + def utcoffset(self, dt): + return None + + +def test_a_datetime_whose_tzinfo_gives_no_offset_is_naive(untouched): + cipher, User = cipher_for(seen=Annotated[datetime.datetime, Timestamp]) + seen = datetime.datetime(2026, 1, 1, tzinfo=NoOffset()) + error = refused(cipher, User(email=SECRET, seen=seen)) + assert error.code == "value_naive_datetime" + + +def test_a_model_of_another_class_is_the_wrong_type(untouched): + # Contact has the same field names, so every getattr would succeed. + cipher, _ = cipher_for(age=Annotated[int, Passthrough]) + Contact = dataclasses.make_dataclass("Contact", [("email", str), ("age", int)]) + error = refused(cipher, Contact(email=SECRET, age=3)) + assert error.code == "value_wrong_type" + assert "model 0 is a Contact" in error.message + + def test_every_model_is_checked_before_any_request(untouched): # The bad value is in the last model of many. cipher, User = cipher_for(age=Annotated[int, Bigint]) diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py index 8cf8442c1..3c6d7d627 100644 --- a/languages/python/tests/test_encrypt_live.py +++ b/languages/python/tests/test_encrypt_live.py @@ -52,6 +52,31 @@ class Contact: email: Annotated[str, TextEq] +class Refused(Exception): + """The model's own error, from its constructor.""" + + +@stackencrypt.model("contacts") +@dataclasses.dataclass +class StrictContact: + """Contact's table and column, with a constructor that refuses a value.""" + + email: Annotated[str, TextEq] + + def __post_init__(self): + if self.email == "bad@example.com": + raise Refused("the model refused this email") + + +def test_an_error_from_the_constructor_reaches_the_caller_unchanged(): + rows = Cipher(Contact).encrypt( + [Contact(email="good@example.com"), Contact(email="bad@example.com")] + ) + + with pytest.raises(Refused, match="the model refused this email"): + Cipher(StrictContact).decrypt(rows) + + @needs_database def test_an_encrypted_row_inserts_with_a_plain_parameter(): psycopg = pytest.importorskip("psycopg") @@ -92,11 +117,25 @@ def test_1001_one_field_models_make_three_key_requests(): assert _engine.request_count() - before == 3 +def test_the_same_value_encrypts_to_a_different_ciphertext_each_time(): + # A reused nonce, or a deterministic seal, would give equal payloads. + users = Cipher(User) + model = User(email="same@example.com", age=3) + + first, second = users.encrypt([model, model]) + again = users.encrypt([model])[0] + + ciphertexts = {json.loads(row["email"])["c"] for row in (first, second, again)} + assert len(ciphertexts) == 3 + + def test_two_ciphers_share_one_engine_client(): Cipher(User).encrypt([User(email="a@example.com", age=1)]) # Read after the first call: an earlier test may have loaded the keyset # more than once, if a load failed and the next call loaded it again. loads = _engine.requests().get("load-keyset") + # A missing counter would make the last check compare None with None. + assert loads is not None and loads >= 1, _engine.requests() Cipher(Contact).encrypt([Contact(email="b@example.com")]) diff --git a/languages/python/tests/test_encrypt_rows.py b/languages/python/tests/test_encrypt_rows.py index e4d5a7ca9..d76a29258 100644 --- a/languages/python/tests/test_encrypt_rows.py +++ b/languages/python/tests/test_encrypt_rows.py @@ -47,6 +47,12 @@ def test_rows_with_no_declared_field_come_back_unchanged_with_no_request(people, assert people.encrypt_rows(rows) == rows +def test_one_row_on_its_own_is_refused_as_one_row(people, untouched): + error = refused(people, {"email": SECRET}) + assert error.code == "value_row_shape" + assert "not one row" in error.message + + def test_a_row_that_is_not_a_mapping_is_refused(people, untouched): error = refused(people, [{"email": SECRET}, ("email", SECRET)]) assert error.code == "value_row_shape" diff --git a/languages/python/tests/test_gate_keyboard_interrupt.py b/languages/python/tests/test_gate_keyboard_interrupt.py index 9b8c4e40f..6ae01cd48 100644 --- a/languages/python/tests/test_gate_keyboard_interrupt.py +++ b/languages/python/tests/test_gate_keyboard_interrupt.py @@ -2,7 +2,16 @@ the GIL is raised only after that call returns. Runs only against a wheel built with the `gate-probe` cargo feature; see -docs/gate-facts/keyboard-interrupt.md. Skipped otherwise. +docs/gate-facts/keyboard-interrupt.md. Skipped otherwise, and CI builds no +such wheel. So this is a check run by hand when the detach or the signal +handling changes, not one CI keeps. + +What the first test shows: Python runs a signal handler only on the main +thread, between bytecodes. So the interrupt cannot be raised while the main +thread is inside `probe()`, whether or not the call released the GIL. The +timing assertion therefore holds by the interpreter's design. What it does +show is that the call returns normally, and that the interrupt is raised +after it, with the extension still usable. """ import sys @@ -30,12 +39,17 @@ def call_then_wait_for_the_interrupt(): def test_interrupt_waits_for_the_detached_call_to_return(): + # Cancelled and joined in `finally`: a timer left running would send + # SIGINT into a later, unrelated test. timer = send_sigint_later(SIGNAL_AT_S) - started = time.monotonic() - with pytest.raises(KeyboardInterrupt): - call_then_wait_for_the_interrupt() - raised_after_s = time.monotonic() - started - timer.join() + try: + started = time.monotonic() + with pytest.raises(KeyboardInterrupt): + call_then_wait_for_the_interrupt() + raised_after_s = time.monotonic() - started + finally: + timer.cancel() + timer.join() # The signal was sent at 0.2 s. If the call had been cut short, the # interrupt would arrive then. It must arrive after the call's full second. @@ -44,8 +58,11 @@ def test_interrupt_waits_for_the_detached_call_to_return(): def test_the_extension_is_usable_after_an_interrupt(): timer = send_sigint_later(SIGNAL_AT_S) - with pytest.raises(KeyboardInterrupt): - call_then_wait_for_the_interrupt() - timer.join() + try: + with pytest.raises(KeyboardInterrupt): + call_then_wait_for_the_interrupt() + finally: + timer.cancel() + timer.join() assert probe(10) >= 10 diff --git a/languages/python/tests/test_readme.py b/languages/python/tests/test_readme.py index 008f27456..a12ff7ab2 100644 --- a/languages/python/tests/test_readme.py +++ b/languages/python/tests/test_readme.py @@ -26,6 +26,31 @@ def test_the_python_blocks_bind_the_names_the_live_check_reads(): assert check_names() <= bound_names(program()) +def test_bound_names_keeps_only_the_top_level_names(): + source = ( + "import os.path as p\n" + "top = 1\n" + "def f():\n" + " inside_function = 2\n" + "class C:\n" + " inside_class = 3\n" + "squares = [n * n for n in range(3)]\n" + "lam = lambda x: x\n" + ) + assert bound_names(source) == {"p", "top", "f", "C", "squares", "lam"} + + +def test_bound_names_reads_except_as_and_a_walrus_in_a_comprehension(): + source = ( + "try:\n" + " pass\n" + "except ValueError as caught:\n" + " pass\n" + "found = [last := n for n in range(3)]\n" + ) + assert bound_names(source) == {"caught", "found", "last"} + + def test_the_readme_has_sql_for_the_live_test_to_run(): assert blocks("sql") @@ -33,6 +58,8 @@ def test_the_readme_has_sql_for_the_live_test_to_run(): def test_the_readme_installs_the_bundle_the_live_tests_install(): # The live task cannot run the README's psql command from the root, so it # runs its own. Both must name the same bundle. - [install] = [block for block in blocks("sh") if "psql" in block] + installs = [block for block in blocks("sh") if "psql" in block] + assert len(installs) == 1, "the README must have exactly one psql install block" + [install] = installs assert BUNDLE in install assert f"../../{BUNDLE}" in MISE.read_text(encoding="utf-8") diff --git a/languages/python/tests/test_redaction.py b/languages/python/tests/test_redaction.py index c47cc67ac..879c7386e 100644 --- a/languages/python/tests/test_redaction.py +++ b/languages/python/tests/test_redaction.py @@ -1,12 +1,15 @@ """``repr`` and ``str`` of a declared class show no sealed value.""" import dataclasses +import functools +import reprlib from typing import Annotated, NamedTuple import attrs import pydantic import pytest -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from sqlalchemy import create_engine, event +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column import stackencrypt from stackencrypt import DeclarationError, Ignore, Passthrough, TextEq @@ -78,6 +81,26 @@ def declare_sqlalchemy(): return MappedUser(email=SECRET, age=42) +def test_an_expired_or_detached_sqlalchemy_column_runs_no_select(): + # After a commit every attribute is expired: reading one runs a SELECT, + # and on a detached instance it raises. repr must do neither. + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + statements = [] + event.listen(engine, "before_cursor_execute", lambda *args: statements.append(args[2])) + expected = "MappedUser(email=, age=)" + with Session(engine) as session: + user = MappedUser(email=SECRET, age=42) + session.add(user) + session.commit() + statements.clear() + + assert repr(user) == expected + assert statements == [] + assert repr(user) == expected + assert statements == [] + + KINDS = [ declare_dataclass, declare_attrs, @@ -102,6 +125,14 @@ def test_repr_and_str_seal_every_encrypted_field(declare): assert list(user.__rich_repr__()) == [("email", SEALED), ("age", 42)] +def test_every_pydantic_print_hook_seals_the_encrypted_fields(): + # Pydantic builds __repr_str__, __pretty__ (devtools' debug()) and + # __rich_repr__ from __repr_args__. + user = declare_pydantic() + assert list(user.__repr_args__()) == [("email", SEALED), ("age", 42)] + assert SECRET not in user.__repr_str__(", ") + + def test_a_hand_written_repr_is_refused(): def build(): @stackencrypt.model("users") @@ -198,17 +229,71 @@ def __init__(self, **kwargs): assert repr(User(email=SECRET, age=9)) == "User(email=, age=9)" -def test_a_hand_written_str_is_refused(): - def build(): +def _repr_from_another_module(self): + return "elsewhere" + + +_repr_from_another_module.__module__ = "somewhere.else" + + +def build_with_a_hand_written_method(form): + if form == "recursive_repr": + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + + @reprlib.recursive_repr() + def __repr__(self): + return f"User({self.email})" + + elif form == "staticmethod": + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + __str__ = staticmethod(lambda: "user") + + elif form == "partialmethod": + @stackencrypt.model("users") @dataclasses.dataclass class User: email: Annotated[str, TextEq] + __str__ = functools.partialmethod(lambda self, text: text, "user") - def __str__(self): - return self.email + else: - assert pytest.raises(DeclarationError, build).value.code == "declaration_repr_present" + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + __repr__ = _repr_from_another_module + + +@pytest.mark.parametrize( + "form", ["recursive_repr", "staticmethod", "partialmethod", "another module"] +) +def test_a_hand_written_method_in_any_form_is_refused(form): + # Only a generator's own output is replaced. Anything else written on the + # class is the user's, and replacing it would drop their code silently. + error = pytest.raises(DeclarationError, build_with_a_hand_written_method, form).value + assert error.code == "declaration_repr_present" + + +@pytest.mark.parametrize("name", ["__str__", "__rich_repr__"]) +def test_each_hand_written_method_is_refused(name): + def method(self): + return self.email + + cls = dataclasses.make_dataclass( + "User", [("email", Annotated[str, TextEq])], namespace={name: method} + ) + error = pytest.raises(DeclarationError, stackencrypt.model("users"), cls).value + assert error.code == "declaration_repr_present" + assert name in str(error) def test_redact_false_keeps_a_hand_written_repr_and_shows_plaintext(): diff --git a/languages/python/tests/test_values.py b/languages/python/tests/test_values.py index a9a4ed9ec..0e7821a23 100644 --- a/languages/python/tests/test_values.py +++ b/languages/python/tests/test_values.py @@ -24,10 +24,12 @@ Boolean, Cipher, DateEq, + DeclarationError, DoubleEq, IntegerEq, InvalidValueError, NumericEq, + QueryError, RealEq, SmallintEq, StashError, @@ -55,18 +57,21 @@ } -def through_encrypt(marker: Any, value: Any) -> None: +def declared(marker: Any) -> Tuple[Cipher, Any]: + """A cipher for a class with one field ``x`` of ``marker``. Built + outside ``pytest.raises``, so a refused declaration fails the test.""" cls = dataclasses.make_dataclass( "Row", [("x", Annotated[PYTHON_TYPE[marker], marker])] ) - Cipher(stackencrypt.model("rows")(cls)).encrypt([cls(x=value)]) + return Cipher(stackencrypt.model("rows")(cls)), cls -def through_eq(marker: Any, value: Any) -> None: - cls = dataclasses.make_dataclass( - "Row", [("x", Annotated[PYTHON_TYPE[marker], marker])] - ) - Cipher(stackencrypt.model("rows")(cls)).eq("x", value) +def through_encrypt(cipher: Cipher, cls: Any, value: Any) -> None: + cipher.encrypt([cls(x=value)]) + + +def through_eq(cipher: Cipher, cls: Any, value: Any) -> None: + cipher.eq("x", value) def calls(cases: List[Tuple[Any, ...]]) -> List[Any]: @@ -85,9 +90,15 @@ def calls(cases: List[Tuple[Any, ...]]) -> List[Any]: (BigintEq, 2**63, "value_out_of_range", "int64"), (BigintEq, -(2**63) - 1, "value_out_of_range", "int64"), (IntegerEq, 2**31, "value_out_of_range", "int32"), + (IntegerEq, -(2**31) - 1, "value_out_of_range", "int32"), + (SmallintEq, 2**15, "value_out_of_range", "int16"), (SmallintEq, -(2**15) - 1, "value_out_of_range", "int16"), (RealEq, 0.1, "value_not_representable", "float32"), + (RealEq, 1e39, "value_not_representable", "float32"), (NumericEq, decimal.Decimal("NaN"), "value_not_representable", "decimal"), + # One past the largest 96-bit mantissa, on each side. + (NumericEq, decimal.Decimal("79228162514264337593543950336"), "value_not_representable", "decimal"), + (NumericEq, decimal.Decimal("-79228162514264337593543950336"), "value_not_representable", "decimal"), (NumericEq, decimal.Decimal("Infinity"), "value_not_representable", "decimal"), (NumericEq, decimal.Decimal("1e-29"), "value_not_representable", "decimal"), (TimestampEq, datetime.datetime(2024, 1, 1), "value_naive_datetime", "timestamp"), @@ -107,8 +118,9 @@ def calls(cases: List[Tuple[Any, ...]]) -> List[Any]: def test_a_value_that_does_not_fit_is_refused_before_any_request( untouched, call, marker, value, code, reason ): + cipher, cls = declared(marker) with pytest.raises(InvalidValueError) as raised: - call(marker, value) + call(cipher, cls, value) assert raised.value.code == code assert raised.value.field == "x" @@ -120,6 +132,8 @@ def test_a_value_that_does_not_fit_is_refused_before_any_request( (BigintEq, 2**63 - 1), (BigintEq, -(2**63)), (IntegerEq, 2**31 - 1), + (IntegerEq, -(2**31)), + (SmallintEq, 2**15 - 1), (SmallintEq, -(2**15)), (RealEq, 0.5), (RealEq, math.nan), @@ -141,7 +155,10 @@ def test_a_value_that_does_not_fit_is_refused_before_any_request( @pytest.mark.parametrize(("call", "marker", "value"), calls(ACCEPTED)) def test_a_value_at_the_edge_passes_every_check(call, marker, value): + cipher, cls = declared(marker) with pytest.raises(StashError) as raised: - call(marker, value) + call(cipher, cls, value) - assert not isinstance(raised.value, InvalidValueError), raised.value + # Only the failure to build the client shows that every check passed. + refused = (InvalidValueError, QueryError, DeclarationError) + assert not isinstance(raised.value, refused), raised.value diff --git a/languages/python/tests_deterministic/test_concurrency.py b/languages/python/tests_deterministic/test_concurrency.py index 24161f487..9baca617f 100644 --- a/languages/python/tests_deterministic/test_concurrency.py +++ b/languages/python/tests_deterministic/test_concurrency.py @@ -7,7 +7,14 @@ live test. pyproject.toml puts `tests/` on the path. """ -from _sync_contract import PER_THREAD, THREADS, User, encrypt_from_threads, user +from _sync_contract import ( + PER_THREAD, + THREADS, + User, + decrypt_from_threads, + encrypt_from_threads, + user, +) from stackencrypt import Cipher @@ -18,3 +25,10 @@ def test_threads_encrypt_at_once_and_every_row_opens(): rows = encrypt_from_threads(cipher, PER_THREAD) assert cipher.decrypt(rows) == [user(n) for n in range(THREADS * PER_THREAD)] + + +def test_threads_decrypt_at_once_and_every_row_opens(): + cipher = Cipher(User) + rows = encrypt_from_threads(cipher, PER_THREAD) + + assert decrypt_from_threads(cipher, rows) == [user(n) for n in range(THREADS * PER_THREAD)] diff --git a/languages/python/tests_deterministic/test_go_rows.py b/languages/python/tests_deterministic/test_go_rows.py index 429ff4463..d458ef27b 100644 --- a/languages/python/tests_deterministic/test_go_rows.py +++ b/languages/python/tests_deterministic/test_go_rows.py @@ -201,6 +201,8 @@ def test_python_opens_the_rows_the_go_sdk_stored(fixture, cls, ids): opened = Cipher(cls).decrypt([stored_row(cls, row) for row in rows]) + # zip stops at the shorter list, and Python 3.9 has no strict=True. + assert len(opened) == len(rows) for row, model in zip(rows, opened): expected = model_of(cls, row) for field in dataclasses.fields(cls): @@ -214,6 +216,7 @@ def test_python_seals_what_the_go_sdk_sealed_apart_from_each_ciphertext(fixture, sealed = Cipher(cls).encrypt([model_of(cls, row) for row in rows]) + assert len(sealed) == len(rows) for row, columns in zip(rows, sealed): for field in sealed_fields(cls): assert without_ciphertext(columns[field.name]) == without_ciphertext( diff --git a/languages/python/tests_deterministic/test_order.py b/languages/python/tests_deterministic/test_order.py new file mode 100644 index 000000000..fc14d7b26 --- /dev/null +++ b/languages/python/tests_deterministic/test_order.py @@ -0,0 +1,18 @@ +"""``decrypt`` returns one instance per row, in the order of the rows. An app +that selects a column outside the model, such as a generated key, pairs it +with the instances by position, as the README says. +""" + +from _sync_contract import User, user + +from stackencrypt import Cipher + + +def test_decrypt_returns_the_instances_in_the_order_of_the_rows(): + cipher = Cipher(User) + rows = cipher.encrypt([user(n) for n in range(6)]) + + assert cipher.decrypt(rows[::-1]) == [user(n) for n in reversed(range(6))] + assert cipher.decrypt_rows(rows[::2]) == [ + {"id": n, "email": user(n).email} for n in range(0, 6, 2) + ] diff --git a/languages/python/tests_deterministic/test_switch.py b/languages/python/tests_deterministic/test_switch.py new file mode 100644 index 000000000..2189f958a --- /dev/null +++ b/languages/python/tests_deterministic/test_switch.py @@ -0,0 +1,22 @@ +"""The refusals of the switch to the deterministic client. + +conftest.py has switched this process once, before any test. A second +switch is refused, and a seed that is not 32 bytes is refused before any +switch is tried. +""" + +import pytest + +from stackencrypt import StashError, _engine + + +def test_a_second_switch_is_refused(): + with pytest.raises(StashError) as raised: + _engine._use_deterministic_keys(bytes(32)) + assert raised.value.code == "stack_test_build" + + +@pytest.mark.parametrize("length", [0, 31, 33]) +def test_a_seed_that_is_not_32_bytes_is_refused(length): + with pytest.raises(ValueError): + _engine._use_deterministic_keys(bytes(length)) diff --git a/languages/python/uv.lock b/languages/python/uv.lock index e00021c0e..f41a44589 100644 --- a/languages/python/uv.lock +++ b/languages/python/uv.lock @@ -460,6 +460,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/7d/87ff065a4d14d8de1f462f851d9cdf903b6e1fc067d0ef3674414da444c0/markupsafe-3.0.4-cp39-cp39-win_arm64.whl", hash = "sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378", size = 14190, upload-time = "2026-10-02T23:07:20.567Z" }, ] +[[package]] +name = "maturin" +version = "1.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/71/eb078174335ff30bde1be2ad54fb6256db4b160eb3efa81163fa30f8c814/maturin-1.16.0.tar.gz", hash = "sha256:13e69ca12542999d8c5467131de72a49bb76e4c284b9a45c5783df7c0fcb3a8a", size = 396641, upload-time = "2026-10-29T14:09:43.096Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/f8/235d4a8b3d61af3c7744582ae9e6ccb8f096e9f68f06fe62d7d03e5d7b2d/maturin-1.16.0-py3-none-linux_armv6l.whl", hash = "sha256:deacbb204a20326862e194c7df4958e0514a0411c9a3ccf228a00067c11b5c7e", size = 10425070, upload-time = "2026-10-29T14:09:06.221Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d5/f49a90d8c489dc5f134654ec91d12a78a51599fdc8c2b5a1bc2391bca0ac/maturin-1.16.0-py3-none-macosx_10_12_universal2.macosx_10_12_x86_64.macosx_11_0_arm64.whl", hash = "sha256:131fcc17ddb3ccc48aa183d2172ebf994c322b858e1b9694fb8744d783df66f6", size = 19839189, upload-time = "2026-10-29T14:09:09.441Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8e/d65633d7c25484cb82735a51e15bc969f9874a13557e2b352d27df574c20/maturin-1.16.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2216d731c9324869d3bff72c0c73009bb6e640b3fa7d340174ae70dc874d1d9f", size = 10271659, upload-time = "2026-10-29T14:09:12.596Z" }, + { url = "https://files.pythonhosted.org/packages/0a/dc/e77f490557cb5e045a75c3d18a7965425ac576235c970dfff4d903eb2c74/maturin-1.16.0-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:837db2f4d93d4899c3ac393fb3cc9eebf35b6002336e5d574f4a4fb6c49fe283", size = 10763361, upload-time = "2026-10-29T14:09:16.11Z" }, + { url = "https://files.pythonhosted.org/packages/fa/eb/535564a4211105fd64611fa4f9437c8925eded26a97d58de84565f5aebfa/maturin-1.16.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:399fadcd4af39532a173e010240c34930eacc1bb1746bb179863481c68bcf1ab", size = 10442856, upload-time = "2026-10-29T14:09:18.64Z" }, + { url = "https://files.pythonhosted.org/packages/45/5b/570e13ce81f103be6609c067a06c5bac08da5af0cf7ea700c9ae43419be9/maturin-1.16.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:e285ec9eb10243d45d10417178cb2eb76e0cbb184d0aa4378b503489fa12ee78", size = 10298796, upload-time = "2026-10-29T14:09:21.385Z" }, + { url = "https://files.pythonhosted.org/packages/88/a6/26b40c31c354359fe4e49398113d018694c7035968fe1b569bb13c3905af/maturin-1.16.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:609228d639c5b888dc41364cbd318547a59f027481f53cba85838f46da4b3f02", size = 10268023, upload-time = "2026-10-29T14:09:23.94Z" }, + { url = "https://files.pythonhosted.org/packages/2b/19/faca1784d917c2eecd498648dff6a0179b299c8142cc448ea5b1183545ec/maturin-1.16.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:7001fd3ac591820d01f2fa041552c367ec587c98649d9f10a817cc7f28e59062", size = 13739038, upload-time = "2026-10-29T14:09:26.662Z" }, + { url = "https://files.pythonhosted.org/packages/36/c1/d67e77aa07100e3b2d4c0d9ef6922646d9c4bab25cc04f91023aefc72418/maturin-1.16.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b0573a146803f24566f5b301a4572a8e49bcab4917175407c730da13b46e134", size = 11100193, upload-time = "2026-10-29T14:09:29.659Z" }, + { url = "https://files.pythonhosted.org/packages/76/44/ccef52f02ab0e1f7258bc8fe0140b4991c6287458ef883af1027f946a253/maturin-1.16.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:c16d3589cbddda9420237fdb1394b1a9b36483c0bce83b23de6118c908a8e940", size = 10663849, upload-time = "2026-10-29T14:09:32.183Z" }, + { url = "https://files.pythonhosted.org/packages/6e/9b/3104f62dbb428f9e26e35cba2c80cfc42ba9bb1dc4968882c19d1bfe594b/maturin-1.16.0-py3-none-win32.whl", hash = "sha256:9e221c047931caa79283cc0c2278904ae1b25880dde3574b0447657d72907689", size = 9124894, upload-time = "2026-10-29T14:09:34.857Z" }, + { url = "https://files.pythonhosted.org/packages/63/b8/c25ce706c4797ee1663bcf1f386ffe91d8a3b3589700a135a204b49bc4b0/maturin-1.16.0-py3-none-win_amd64.whl", hash = "sha256:9d43a2adf51a2794e907a8f7dbad06eb114d5edb0909c73c1fa9040f15e584e9", size = 10575645, upload-time = "2026-10-29T14:09:37.864Z" }, + { url = "https://files.pythonhosted.org/packages/eb/74/f69aba8b22539e5ed58323d4a6ee974afd9edda238489c73926cc0fc9d51/maturin-1.16.0-py3-none-win_arm64.whl", hash = "sha256:95f34ce53249fb7095ff4f2aff64d2562c511d6edf3e216c15dbffcd82b72b4f", size = 9886475, upload-time = "2026-10-29T14:09:40.603Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -1225,6 +1249,11 @@ test = [ { name = "sqlalchemy", version = "2.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] +[package.dev-dependencies] +build = [ + { name = "maturin" }, +] + [package.metadata] requires-dist = [ { name = "alembic", marker = "extra == 'test'", specifier = ">=1.13" }, @@ -1236,6 +1265,9 @@ requires-dist = [ ] provides-extras = ["test"] +[package.metadata.requires-dev] +build = [{ name = "maturin", specifier = ">=1.9,<2.0" }] + [[package]] name = "tomli" version = "2.5.0" diff --git a/packages/eql/crates/eql-codegen/src/py_eql.rs b/packages/eql/crates/eql-codegen/src/py_eql.rs index 4a074273b..069933e7c 100644 --- a/packages/eql/crates/eql-codegen/src/py_eql.rs +++ b/packages/eql/crates/eql-codegen/src/py_eql.rs @@ -64,15 +64,40 @@ fn py_tuple(items: &[&str]) -> String { /// The class docstring: the summary and the status line the Go emitter /// writes too ([`Row::summary`], [`Row::status`]). fn docstring(row: &Row) -> String { - let text = format!( + py_docstring(&format!( "The EQL type {summary}.\n\n {status}\n ", summary = row.summary(), status = row.status(), - ); + )) +} + +/// `text` as a Python docstring literal. A backslash is doubled, so Python +/// reads it as written. Text with `"""` in it is refused: it would end the +/// docstring. +fn py_docstring(text: &str) -> String { assert!(!text.contains("\"\"\""), "catalog text ends the docstring"); format!("\"\"\"{}\"\"\"", text.replace('\\', "\\\\")) } +/// The names a catalog row may not take as its class name: Python keywords +/// that are names, and the module's own names, which a class of the same +/// name would replace. +const RESERVED_NAMES: [&str; 5] = ["Marker", "None", "True", "False", "TYPES"]; + +/// `name` as a Python class name. A catalog name is written straight into +/// `class {name}(Marker):` and into `TYPES`, so a name that is not a plain +/// class name, or that replaces one of the module's names, is refused here +/// rather than in the Python job. +fn py_ident(name: &str) -> &str { + assert!( + name.starts_with(|c: char| c.is_ascii_uppercase()) + && name.chars().all(|c| c.is_ascii_alphanumeric()) + && !RESERVED_NAMES.contains(&name), + "catalog name {name:?} is not a usable Python class name" + ); + name +} + fn class(out: &mut String, row: &Row) { let (query, query_sql_domain) = match &row.query { Some((query, domain)) => (Some(query.as_str()), Some(domain.as_str())), @@ -91,7 +116,7 @@ fn class(out: &mut String, row: &Row) { query_sql_domain = {query_sql_domain}\n \ producible = {producible}\n \ reason = {reason}\n", - name = row.name, + name = py_ident(&row.name), doc = docstring(row), name_lit = py_str(&row.name), family = py_str(row.family), @@ -136,7 +161,7 @@ pub fn render_py_eql() -> String { } out.push_str("\n\nTYPES = (\n"); for row in &rows { - let _ = writeln!(out, " {},", row.name); + let _ = writeln!(out, " {},", py_ident(&row.name)); } out.push_str(")\n"); out @@ -161,6 +186,42 @@ mod tests { assert_eq!(py_str(r#"a "b" \c"#), r#""a \"b\" \\c""#); } + #[test] + #[should_panic(expected = "control character")] + fn a_string_with_a_control_character_is_refused() { + py_str("a\nb"); + } + + #[test] + fn an_absent_string_is_none() { + assert_eq!(py_opt(None), "None"); + assert_eq!(py_opt(Some("eq")), r#""eq""#); + } + + #[test] + fn a_docstring_doubles_a_backslash() { + assert_eq!(py_docstring(r"a \b"), r#""""a \\b""""#); + } + + #[test] + #[should_panic(expected = "ends the docstring")] + fn a_docstring_with_three_quotes_is_refused() { + py_docstring(r#"a """ b"#); + } + + #[test] + fn a_class_name_is_a_plain_capitalised_identifier() { + assert_eq!(py_ident("TextEq"), "TextEq"); + for bad in [ + "None", "True", "Marker", "TYPES", "textEq", "Text_Eq", "1Text", "", + ] { + assert!( + std::panic::catch_unwind(|| py_ident(bad)).is_err(), + "{bad:?} was accepted" + ); + } + } + #[test] fn tuples_have_python_syntax_for_every_length() { assert_eq!(py_tuple(&[]), "()"); diff --git a/packages/eql/crates/eql-codegen/tests/cli.rs b/packages/eql/crates/eql-codegen/tests/cli.rs index 494cc8672..a2f2ecd9b 100644 --- a/packages/eql/crates/eql-codegen/tests/cli.rs +++ b/packages/eql/crates/eql-codegen/tests/cli.rs @@ -223,14 +223,18 @@ fn select_subcommand_drops_files_confined_to_the_other_version() { /// `go-eql` and `py-eql` write outside the EQL subtree, so an /// `EQL_CODEGEN_OUT_ROOT` override would send them outside the throwaway tree. -/// Each refuses the override with exit 2, before writing anything. +/// Each refuses the override with exit 2, before writing anything. The +/// override is two folders deep, so the emitters' `../../` paths would land +/// inside the throwaway tree, where the test looks for them. #[test] fn cross_tree_emitters_refuse_the_output_root_override() { - let out_root = tempdir(); + let tree = tempdir(); + let out_root = tree.0.join("one").join("two"); + std::fs::create_dir_all(&out_root).expect("create the nested output root"); for command in ["go-eql", "py-eql"] { let out = Command::new(bin()) .arg(command) - .env("EQL_CODEGEN_OUT_ROOT", out_root.0.as_os_str()) + .env("EQL_CODEGEN_OUT_ROOT", out_root.as_os_str()) .output() .unwrap_or_else(|e| panic!("run eql-codegen {command}: {e}")); let stderr = String::from_utf8_lossy(&out.stderr); @@ -240,6 +244,15 @@ fn cross_tree_emitters_refuse_the_output_root_override() { stderr.contains("EQL_CODEGEN_OUT_ROOT"), "{command}: {stderr}" ); + for path in [ + eql_codegen::go_eql::GO_EQL_PATH, + eql_codegen::py_eql::PY_EQL_PATH, + ] { + assert!( + !out_root.join(path).exists(), + "{command} wrote {path} before it refused" + ); + } } } diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index c1962a854..82647d3cc 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -811,17 +811,22 @@ pub struct FromEnv; /// workspace's `secretkey.json` in the profile directory. A profile directory /// that cannot be resolved is not an error here — env-only setups (CI) have /// none — it just leaves the environment as the only source. +/// +/// This is the provider [`StackCipherBuilder::init`] uses. It is public so +/// that a client built over its own connection, as the Python SDK builds one +/// to count its requests, finds the client key the same way. #[cfg(all(feature = "http", not(target_arch = "wasm32")))] -fn client_key_provider() -> FallbackKeyProvider { +pub fn client_key_provider() -> impl KeyProvider { FallbackKeyProvider::new( EnvKeyProvider, ProfileClientKey(ProfileStore::resolve(None).ok()), ) } -/// wasm32 has no filesystem, so no profile: the environment is the only source. +/// The client key from the environment: wasm32 has no filesystem, so no +/// profile, and the environment is the only source. #[cfg(all(feature = "http", target_arch = "wasm32"))] -fn client_key_provider() -> EnvKeyProvider { +pub fn client_key_provider() -> impl stack_kms::KeyProvider { EnvKeyProvider } diff --git a/packages/stack-encrypt/src/lib.rs b/packages/stack-encrypt/src/lib.rs index d45570e82..34fff1659 100644 --- a/packages/stack-encrypt/src/lib.rs +++ b/packages/stack-encrypt/src/lib.rs @@ -282,6 +282,8 @@ pub mod sem; pub mod target; pub mod value; +#[cfg(feature = "http")] +pub use cipher::client_key_provider; pub use cipher::{ Error, FromEnv, LeafBytesError, PendingStackCipherText, SealedValue, StackCipher, StackCipherBuilder, StackCipherText, diff --git a/packages/stack-kms/src/builder.rs b/packages/stack-kms/src/builder.rs index 7d1bca258..7a277a724 100644 --- a/packages/stack-kms/src/builder.rs +++ b/packages/stack-kms/src/builder.rs @@ -227,7 +227,7 @@ impl StackKmsBuilder { fn build_opts(self) -> Result<(ClientOpts, C, S), StackKmsBuilderError> { let mut connection = self.connection; if connection.base_url().is_none() { - if let Some(endpoint) = Self::base_url_from_env()? { + if let Some(endpoint) = endpoint_from_env()? { connection = connection.with_base_url(endpoint); } } @@ -241,24 +241,34 @@ impl StackKmsBuilder { Ok((opts, self.credentials, self.client_key)) } +} - /// Resolve the ZeroKMS endpoint from the `CS_ZEROKMS_HOST` environment - /// variable (or legacy `CS_VITUR_HOST`). The first variable that is set - /// decides: an unusable value is an error, not a fall-through. - fn base_url_from_env() -> Result, StackKmsBuilderError> { - use crate::vars::CS_ZEROKMS_HOST; - - for env_var in CS_ZEROKMS_HOST { - if let Ok(value) = std::env::var(env_var) { - return value - .parse() - .map(Some) - .map_err(|source| StackKmsBuilderError::InvalidEndpoint { env_var, source }); - } +/// The ZeroKMS endpoint from the environment, as [`StackKmsBuilder`] reads +/// it: `CS_ZEROKMS_HOST`, or the legacy `CS_VITUR_HOST`. The first variable +/// that is set decides: an unusable value is an error, not a fall-through. +/// `None` when neither is set, which leaves the endpoint to the access +/// token's services claim. +/// +/// Public so that a client built over its own connection, as the Python +/// SDK builds one to count its requests, reads the endpoint the same way. +/// +/// # Errors +/// +/// [`StackKmsBuilderError::InvalidEndpoint`], naming the variable, when its +/// value is not a usable endpoint. +pub fn endpoint_from_env() -> Result, StackKmsBuilderError> { + use crate::vars::CS_ZEROKMS_HOST; + + for env_var in CS_ZEROKMS_HOST { + if let Ok(value) = std::env::var(env_var) { + return value + .parse() + .map(Some) + .map_err(|source| StackKmsBuilderError::InvalidEndpoint { env_var, source }); } - - Ok(None) } + + Ok(None) } impl StackKmsBuilder @@ -370,7 +380,7 @@ mod tests { const LEGACY: &str = "CS_VITUR_HOST"; fn from_env() -> Result, StackKmsBuilderError> { - StackKmsBuilder::::base_url_from_env() + endpoint_from_env() } #[test] diff --git a/packages/stack-kms/src/lib.rs b/packages/stack-kms/src/lib.rs index ee484f39e..30a28fdfb 100644 --- a/packages/stack-kms/src/lib.rs +++ b/packages/stack-kms/src/lib.rs @@ -95,7 +95,7 @@ pub mod vars; // Builder (configures the default HTTP transport) #[cfg(feature = "http")] -pub use builder::{StackKmsBuilder, StackKmsBuilderError, WithKeyProvider}; +pub use builder::{endpoint_from_env, StackKmsBuilder, StackKmsBuilderError, WithKeyProvider}; // Clients pub use client::{ diff --git a/scripts/__tests__/cipherstash-client-parity.test.mjs b/scripts/__tests__/cipherstash-client-parity.test.mjs index 305ab4f65..7bc90cf2e 100644 --- a/scripts/__tests__/cipherstash-client-parity.test.mjs +++ b/scripts/__tests__/cipherstash-client-parity.test.mjs @@ -1,6 +1,7 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' +import { lockedPackages } from './lib/cargo-lock.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' /** @@ -32,28 +33,6 @@ import { REPO_ROOT } from './lib/repo-root.mjs' const EQL_LOCK = 'packages/eql/Cargo.lock' const FFI_LOCK = 'languages/typescript/packages/protect-ffi/Cargo.lock' -/** - * Every `name` package a `Cargo.lock` records, as its version, source and - * checksum, sorted by version. The version alone is not enough: a git source - * or a `[patch.crates-io]` fork can keep the version number while resolving - * different code, and the source and checksum are what tell them apart. - */ -export function lockedPackages(source, name) { - const field = (block, key) => - new RegExp(`^${key} = "([^"]*)"$`, 'm').exec(block)?.[1] ?? null - return source - .split(/^\[\[package\]\]$/m) - .slice(1) - .filter((block) => new RegExp(`^name = "${name}"$`, 'm').test(block)) - .map((block) => ({ - version: field(block, 'version'), - source: field(block, 'source'), - checksum: field(block, 'checksum'), - })) - .filter((pkg) => pkg.version) - .sort((a, b) => a.version.localeCompare(b.version)) -} - const read = (file) => readFileSync(join(REPO_ROOT, file), 'utf8') describe('lockedPackages', () => { diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index aa73d3bae..79755e0b4 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -233,6 +233,14 @@ describe('a `mise run` counts as an EQL task only from inside packages/eql', () expect(eqlRunBodies(wf)).toEqual([]) }) + it('does not read a sibling folder with the same prefix as packages/eql', () => { + const wf = workflow({ + sibling: { steps: [step('mise run test:sqlx', 'packages/eql-bench')] }, + slash: { steps: [step('mise run test:schema', 'packages/eql/')] }, + }) + expect(eqlRunBodies(wf)).toEqual(['mise run test:schema']) + }) + it('strips shell comment lines, so a mention in a comment is not a run', () => { const wf = workflow({ eql: { @@ -578,23 +586,34 @@ const RUST_CACHE_STEPS = workflowFiles().flatMap((relPath) => { }) /** - * Whether a rust-cache step names the root of another Cargo workspace: a - * directory outside the EQL tree with its own `Cargo.lock`. A job that builds - * both EQL and that workspace (tests-python.yml's `live` builds the Python - * wheel and the EQL bundle) caches each in its own step, and the other - * workspace's step is right where it is. A member of the EQL workspace, or - * the monorepo root, is not another workspace, so this check still catches - * both. + * The workspaces a rust-cache step names: one per line of `workspaces:`, as + * rust-cache reads it. A job that builds EQL and another workspace + * (tests-python.yml's `live` builds the Python wheel and the EQL bundle) + * lists both in one step. Two steps would each prune the shared cargo + * registry to their own crates before they save. + */ +const workspacesOf = (step) => + String(step?.with?.workspaces ?? '') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + +/** + * Whether `dir` is the root of another Cargo workspace: a directory outside + * the EQL tree with its own `Cargo.lock`. A member of the EQL workspace, or + * the monorepo root, is not another workspace, so the check below still + * catches both. */ +const isAnotherWorkspace = (dir) => + dir !== '.' && + dir !== CARGO_WORKSPACE && + !dir.startsWith(`${CARGO_WORKSPACE}/`) && + existsSync(join(REPO_ROOT, dir, 'Cargo.lock')) + +/** A step that caches other workspaces only, and not EQL's. */ const namesAnotherWorkspace = ({ step }) => { - const dir = step?.with?.workspaces - return ( - typeof dir === 'string' && - dir !== '.' && - dir !== CARGO_WORKSPACE && - !dir.startsWith(`${CARGO_WORKSPACE}/`) && - existsSync(join(REPO_ROOT, dir, 'Cargo.lock')) - ) + const dirs = workspacesOf(step) + return dirs.length > 0 && dirs.every(isAnotherWorkspace) } /** The rust-cache steps in EQL's jobs that cache the EQL workspace. */ @@ -642,9 +661,15 @@ describe('the shared Rust cache is pointed and saved correctly', () => { // root workspace, and rust-cache's default (`. -> target`) is right there. // Held non-empty, so a change to `isEqlJob` cannot empty the check. expect(EQL_RUST_CACHE_STEPS.length).toBeGreaterThan(0) - const wrong = EQL_RUST_CACHE_STEPS.filter( - ({ step }) => step?.with?.workspaces !== CARGO_WORKSPACE, - ).map((entry) => + // The step names the EQL root, and any other entry is another + // workspace's root, never an EQL member or the monorepo root. + const wrong = EQL_RUST_CACHE_STEPS.filter(({ step }) => { + const dirs = workspacesOf(step) + return ( + !dirs.includes(CARGO_WORKSPACE) || + !dirs.every((dir) => dir === CARGO_WORKSPACE || isAnotherWorkspace(dir)) + ) + }).map((entry) => stepLabel(entry, `workspaces: ${entry.step?.with?.workspaces}`), ) expect( diff --git a/scripts/__tests__/lib/cargo-lock.mjs b/scripts/__tests__/lib/cargo-lock.mjs index e056c2ffe..fcce47d16 100644 --- a/scripts/__tests__/lib/cargo-lock.mjs +++ b/scripts/__tests__/lib/cargo-lock.mjs @@ -15,3 +15,25 @@ export function pathPackageNames(lock) { return name }) } + +/** + * Every `name` package a `Cargo.lock` records, as its version, source and + * checksum, sorted by version. The version alone is not enough: a git source + * or a `[patch.crates-io]` fork can keep the version number while resolving + * different code, and the source and checksum are what tell them apart. + */ +export function lockedPackages(source, name) { + const field = (block, key) => + new RegExp(`^${key} = "([^"]*)"$`, 'm').exec(block)?.[1] ?? null + return source + .split(/^\[\[package\]\]$/m) + .slice(1) + .filter((block) => new RegExp(`^name = "${name}"$`, 'm').test(block)) + .map((block) => ({ + version: field(block, 'version'), + source: field(block, 'source'), + checksum: field(block, 'checksum'), + })) + .filter((pkg) => pkg.version) + .sort((a, b) => a.version.localeCompare(b.version)) +} diff --git a/scripts/__tests__/python-ci.test.mjs b/scripts/__tests__/python-ci.test.mjs index b020e43f5..2ff648195 100644 --- a/scripts/__tests__/python-ci.test.mjs +++ b/scripts/__tests__/python-ci.test.mjs @@ -117,6 +117,14 @@ describe('tests-python.yml', () => { ).toEqual([]) }) + it('finds the stack crates as path dependencies, so the filter check is not empty', () => { + // A lockfile parser that finds nothing would leave the check above with + // no tree to look for, and it would pass. + expect(pathDependencyTrees()).toEqual( + expect.arrayContaining(['packages/stack-encrypt', 'packages/stack-kms']), + ) + }) + it('runs on pull requests touching a file the core crate compiles by #[path]', () => { const paths = triggers(readWorkflow(WORKFLOW)).pull_request?.paths ?? [] const files = pathModuleFiles() @@ -208,6 +216,12 @@ describe('tests-python.yml', () => { const compiles = (step) => /\bcargo\b|\bmise run\b/.test(step.run ?? '') || /^PyO3\/maturin-action@/.test(step.uses ?? '') + // The scan must find the jobs that compile, or the check below passes + // over none: a wheel step in another form would drop out of it. + const compiling = Object.entries(wf.jobs) + .filter(([, job]) => job.steps.some(compiles)) + .map(([id]) => id) + expect(compiling.sort()).toEqual(['live', 'python', 'wheels']) const missing = Object.entries(wf.jobs) .filter(([, job]) => job.steps.some(compiles)) .filter( @@ -223,6 +237,74 @@ describe('tests-python.yml', () => { ).toEqual([]) }) + it('builds the deterministic test extension with the features of the normal build', () => { + // `--features` replaces pyproject.toml's list. A feature added there and + // not here would leave tests_deterministic/ testing another extension. + const pyproject = read(`${PYTHON_ROOT}/pyproject.toml`) + const declared = pyproject.match(/^features = \[([^\]]*)\]/m) + expect( + declared, + 'pyproject.toml has a [tool.maturin] features list', + ).not.toBeNull() + const normal = [...declared[1].matchAll(/"([^"]+)"/g)].map((m) => m[1]) + const task = read(`${PYTHON_ROOT}/mise.toml`).match(/--features ([^\s"]+)/) + expect(task, 'test:deterministic passes --features').not.toBeNull() + expect(task[1].split(',').sort()).toEqual( + [...normal, 'deterministic-kms'].sort(), + ) + }) + + it('runs the parity test that refuses a hand edit to _eql.py', () => { + // The one check that fails when someone edits the generated + // `_eql.py` by hand: test-eql.yml does not start on a change under + // languages/python. + const wf = readWorkflow(WORKFLOW) + const runs = Object.values(wf.jobs) + .flatMap((job) => job.steps) + .map((step) => step.run ?? '') + .filter((run) => + [ + 'cargo test', + '--locked', + '-p eql-codegen', + '--test py_eql_parity', + ].every((part) => run.includes(part)), + ) + expect( + runs, + "steps that run eql-codegen's py_eql_parity test", + ).toHaveLength(1) + }) + + it('builds from uv.lock, with no tool resolved outside it', () => { + // `uv sync --locked` refuses a lock that does not match pyproject.toml; + // without it, uv would update the lock and the build would pass. Each + // `uv pip install` must take nothing from the index: `--no-deps` for + // the package's dependencies, `--no-build-isolation` for maturin. + expect(existsSync(join(REPO_ROOT, PYTHON_ROOT, 'uv.lock'))).toBe(true) + const miseToml = read(`${PYTHON_ROOT}/mise.toml`) + const build = miseToml.match( + /^\[tasks\.build\]\n([\s\S]*?)(?=^\[|(?![\s\S]))/m, + ) + expect(build, 'mise.toml has a [tasks.build]').not.toBeNull() + const lines = build[1] + .split('\n') + .filter((line) => !line.trim().startsWith('#')) + const syncs = lines.filter((line) => /\buv sync\b/.test(line)) + expect(syncs, 'uv sync lines in the build task').toHaveLength(1) + expect(syncs[0]).toContain('--locked') + expect(syncs[0]).toContain('--group build') + const installs = lines.filter((line) => /\buv pip install\b/.test(line)) + expect( + installs.length, + 'uv pip install lines in the build task', + ).toBeGreaterThan(0) + for (const line of installs) { + expect(line).toContain('--no-deps') + expect(line).toContain('--no-build-isolation') + } + }) + it('runs every task of the Python workspace, and the two cargo lints, from that folder', () => { // "A check nothing invokes reads exactly like a check that passes" // (crates-ci.test.mjs). That file scans the root mise.toml and the crate @@ -255,8 +337,25 @@ describe('tests-python.yml', () => { new RegExp(`(^|[\\s;&|])${command}(\\s|$)`, 'm').test(run), ) + // A local shortcut CI runs in parts. `test:live` builds, then runs + // `test:live:run`. The live job runs the builds and `test:live:run` as + // separate steps, so that no build runs with the CS_* credentials set. + const localOnly = { 'test:live': 'mise run test:live:run' } + for (const [task, runs] of Object.entries(localOnly)) { + const body = miseToml.match( + new RegExp( + `^\\[tasks\\."${task}"\\]\\n([\\s\\S]*?)(?=^\\[|(?![\\s\\S]))`, + 'm', + ), + ) + expect(body, `${task} is still a task`).not.toBeNull() + expect(body[1], `${task} still runs ${runs}`).toContain(runs) + } + const wanted = [ - ...tasks.filter((t) => !dependedOn.has(t)).map((t) => `mise run ${t}`), + ...tasks + .filter((t) => !dependedOn.has(t) && !(t in localOnly)) + .map((t) => `mise run ${t}`), 'cargo fmt --all --check', 'cargo clippy', ] diff --git a/scripts/__tests__/python-lock-parity.test.mjs b/scripts/__tests__/python-lock-parity.test.mjs new file mode 100644 index 000000000..97eb63254 --- /dev/null +++ b/scripts/__tests__/python-lock-parity.test.mjs @@ -0,0 +1,83 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { lockedPackages } from './lib/cargo-lock.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' + +/** + * The Python SDK's workspace and the root workspace must resolve the same + * crates that decide what a stored value is: the encryption, the order + * terms, and the key protocol. + * + * `languages/python` is its own Cargo workspace, with its own Cargo.lock. Its + * engine builds the stack-* crates by path, but each lock picks its own + * versions of the crates from crates.io. The root workspace's tests run the + * stack crates against the root lock. If the Python lock resolves another + * version of one of these crates, the Python engine runs code those tests + * never ran, and it can write values the other SDKs do not read the same way. + * + * Each crate is compared by version, source and checksum, as + * `cipherstash-client-parity.test.mjs` compares its crates. Other shared + * crates, such as chrono, may differ. + * + * The fix for a failure is to seed the Python lock from the root one again, + * as AGENTS.md describes. + * + * Node builtins only: no cargo on `pnpm run test:scripts` (see AGENTS.md). + */ + +const ROOT_LOCK = 'Cargo.lock' +const PYTHON_LOCK = 'languages/python/Cargo.lock' + +const read = (file) => readFileSync(join(REPO_ROOT, file), 'utf8') +const root = read(ROOT_LOCK) +const python = read(PYTHON_LOCK) + +/** Every package name a lock records. */ +const names = (lock) => + new Set([...lock.matchAll(/^name = "([^"]+)"$/gm)].map((m) => m[1])) + +/** + * The crates that decide what a stored value is. Every vitaminc crate is + * one: vitaminc is the encryption and the order terms. The rest are named: + * recipher, and the two order schemes, which the root Cargo.toml pins with + * `=`; cts-common and zerokms-protocol, the key protocol. + */ +const VALUE_CRATES = [ + ...[...names(root)].filter((name) => name.startsWith('vitaminc')).sort(), + 'recipher', + 'ore-rs', + 'clww-ore', + 'cts-common', + 'zerokms-protocol', +] + +const RESEED = + 'Seed it from the root lock again: copy Cargo.lock over ' + + 'languages/python/Cargo.lock, and run `cargo metadata` in languages/python. ' + + 'If cargo moves this crate to a newer version, move it back with ' + + '`cargo update -p @ --precise `.' + +describe('the Python lock and the root lock', () => { + it('both name every crate the comparison checks', () => { + // The floor: a crate that leaves either lock would make its comparison + // below compare two empty lists, and pass. + expect( + VALUE_CRATES.filter((name) => name.startsWith('vitaminc')).length, + ).toBeGreaterThan(0) + const missing = VALUE_CRATES.filter( + (name) => !names(root).has(name) || !names(python).has(name), + ) + expect(missing, 'crates missing from one of the two locks').toEqual([]) + }) + + it.each(VALUE_CRATES)('resolve the same %s', (name) => { + const inRoot = lockedPackages(root, name) + const inPython = lockedPackages(python, name) + expect( + inPython, + `${PYTHON_LOCK} resolves ${name} ${JSON.stringify(inPython)}, and ` + + `${ROOT_LOCK} resolves ${JSON.stringify(inRoot)}. ${RESEED}`, + ).toEqual(inRoot) + }) +}) diff --git a/scripts/__tests__/workflows-lib.test.mjs b/scripts/__tests__/workflows-lib.test.mjs new file mode 100644 index 000000000..691209edf --- /dev/null +++ b/scripts/__tests__/workflows-lib.test.mjs @@ -0,0 +1,25 @@ +import yaml from 'js-yaml' +import { describe, expect, it } from 'vitest' +import { triggers } from './lib/workflows.mjs' + +// Seven workflow guards read their triggers through this helper. A change +// that drops a trigger would make each of them check fewer workflows, and +// pass. +describe('triggers()', () => { + it('reads `on:` under the YAML 1.1 `true` key and under a quoted key', () => { + expect(triggers(yaml.load('on:\n push: {}'))).toHaveProperty('push') + expect(triggers(yaml.load("'on':\n push: {}"))).toHaveProperty('push') + }) + + it('keeps a trigger declared with no value', () => { + expect( + 'workflow_dispatch' in triggers(yaml.load('on:\n workflow_dispatch:')), + ).toBe(true) + }) + + it('gives {} for a missing or non-object `on:`', () => { + expect(triggers(yaml.load('name: x'))).toEqual({}) + expect(triggers(yaml.load('on: push'))).toEqual({}) + expect(triggers(undefined)).toEqual({}) + }) +})