From 8191043870699734767c761355bc9340b3d12c1d Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 14:52:26 +0900 Subject: [PATCH 01/21] fix(python): keep the test key source out of wheels, and tighten CI A release build with the deterministic-kms feature now fails to compile. That feature exports _use_deterministic_keys. That function sets every key in the process from a fixed seed. Only the dev-profile test build uses it. The wheel import step also fails if a wheel exports it. tests-python.yml: - The live job skips Dependabot's pull requests. They get no Actions secrets, so require-cs-secrets failed on every one. - An example-only change no longer starts the workflow. CI does not run the example's tests. - .cargo/** and EQL's Cargo.toml and Cargo.lock now start it. Cargo reads the first for this workspace. The EQL build uses the other two. - A pull request's wheel records the PR's head commit. GitHub's test merge commit is on no branch, so a bug report could not name it. - maturin is fixed at v1.16.0, inside pyproject.toml's range. - wheels-summary runs when a leg fails, so the passing legs still get their build-time row. Addresses review threads 1513-F5, 1499-F2, 1499-F12, 1621-F5, and 1406-F1, F2, F3, F5. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests-python.yml | 28 ++++++++++++++++++++++--- languages/python/crates/core/src/lib.rs | 10 +++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index 1573cefaf..6ab8856d5 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -27,6 +27,9 @@ on: pull_request: paths: - languages/python/** + # The example has its own lock and its own tests, which CI does not + # run. A change only there must not start this workflow. + - '!languages/python/examples/**' # The crates the workspace compiles by path, in the order Cargo.lock # lists them. The last two are what stack-kms's `profile` feature pulls # in. The root manifest @@ -38,6 +41,9 @@ on: - packages/stack-auth/** - packages/stack-profile/** - Cargo.toml + # Cargo reads the root .cargo/config.toml for this workspace too. A + # [patch] or [build] change there changes what this workspace compiles. + - .cargo/** # The EQL codegen and the catalog it reads. The SDK's marker names are # generated from the catalog (#1385). A change to either must rerun the # suite that reads them. @@ -59,6 +65,9 @@ on: # Go workflow's eql-postgres job does, and runs the secrets pre-flight. - packages/eql/src/** - packages/eql/mise.toml + # The EQL build compiles eql-codegen, whose versions these two choose. + - packages/eql/Cargo.toml + - packages/eql/Cargo.lock - packages/eql/tasks/** - .github/actions/require-cs-secrets/** - .github/workflows/tests-python.yml @@ -206,8 +215,9 @@ jobs: live: name: "Python live: encrypt into PostgreSQL with EQL v4" # A fork's pull request has no secrets: skip it rather than fail it, as - # the integration workflows do. - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + # the integration workflows do. Dependabot's pull requests come from this + # repository but get no Actions secrets either, so they skip too. + if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' }} runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 45 env: @@ -439,9 +449,14 @@ jobs: # wheel names its engine the same way. The checkout is clean here, so # the `-dirty` check the script skips with the override has nothing to # add. python-ci.test.mjs holds the variable name to build.rs. + # On a pull request, HEAD is GitHub's test merge commit, which is on no + # branch. The pull request's head commit is the one a bug report can + # name, so the wheel records that one. - name: Read the engine commit id: engine - run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + env: + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: echo "commit=$COMMIT" >> "$GITHUB_OUTPUT" # `--locked`: the Cargo.lock in the tree is the one built, as in every # other check of this workspace. `--out dist` puts the wheel where the @@ -459,6 +474,9 @@ jobs: target: ${{ matrix.target }} manylinux: ${{ matrix.manylinux }} rust-toolchain: ${{ steps.rust.outputs.version }} + # A fixed maturin, inside pyproject.toml's range. Without it the + # action downloads the newest release on every run. + maturin-version: v1.16.0 docker-options: -e STACKENCRYPT_ENGINE_COMMIT # The build time is the number this job exists to record. The spec's @@ -508,6 +526,8 @@ jobs: set -euo pipefail python -m pip install --no-index --no-deps languages/python/dist/*.whl python -c 'import stackencrypt as s; print("stackencrypt", s.__version__, "engine", s.__engine__)' + # A wheel must not carry the deterministic key source. + python -c 'from stackencrypt import _engine; assert not hasattr(_engine, "_use_deterministic_keys"), "this wheel has the deterministic-kms test feature"' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: @@ -530,6 +550,8 @@ jobs: wheels-summary: name: one wheel per platform, build time per platform needs: [wheels] + # Run when a leg fails too, so the table still shows the legs that built. + if: ${{ !cancelled() }} runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: diff --git a/languages/python/crates/core/src/lib.rs b/languages/python/crates/core/src/lib.rs index fd2faf5fe..2d1981b16 100644 --- a/languages/python/crates/core/src/lib.rs +++ b/languages/python/crates/core/src/lib.rs @@ -24,6 +24,16 @@ compile_error!( "stackencrypt-core needs its `eql` feature: every build of the SDK holds the EQL types" ); +// The deterministic key source takes every key from a fixed seed. A wheel +// built with it would export `_use_deterministic_keys`, which switches every +// key in the process to that seed. Only the test build uses it, and that +// build is the dev profile. A release build with the feature is refused here, +// so a release wheel cannot carry it. +#[cfg(all(feature = "deterministic-kms", not(debug_assertions)))] +compile_error!( + "the `deterministic-kms` feature is for test builds only: a release build must not carry it" +); + // The EQL resolver: the Go guest's own file, compiled here too, so the two // SDKs run one resolver and cannot drift apart. It switches on this crate's // `eql` feature. Its tests run here as well as in the guest. `HOLDS_EQL` is From 6b5b6875d3ffe409d052efe6c3c848d06d70c819 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 15:03:53 +0900 Subject: [PATCH 02/21] fix(python): refuse hand-written methods and read declarations correctly The decorator replaced a hand-written __repr__ or __str__ with no error in four cases: a method wrapped by reprlib.recursive_repr, a staticmethod, a partialmethod, or a function from another module. Now the decorator replaces only a method that dataclasses, attrs, or NamedTuple generated. It refuses anything else on the class with declaration_repr_present. The field reader had five faults: - It refused a correctly placed marker around a generic value type, such as re.Pattern[str]. - It read dunder annotations, such as __tablename__, as fields. - An AttributeError or a SyntaxError from a hint escaped with no field name. - Before Python 3.14, it named the wrong field after a string ClassVar hint. - It did not refuse a required Ignore field with a Pydantic alias or an attrs private name. The fixes: - Only a SQLAlchemy wrapper is refused inside the marker's Annotated. - Dunder names are skipped. - The four hint errors give declaration_hints_unresolved. - Each hint is resolved on a probe class, not a probe function. - An Ignore field is looked up by its constructor name. - A Pydantic dataclass's config is read from __pydantic_config__. Addresses review threads 1453-F1, F2, 1433-F1, F2, F3, F5, and 1437-F1, F2, F3. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .../python/stackencrypt/_declaration.py | 118 +++++++++++----- .../python/python/stackencrypt/_redact.py | 63 +++++---- languages/python/tests/_declaration_cases.py | 133 +++++++++++++++++- languages/python/tests/test_redaction.py | 56 ++++++++ 4 files changed, 311 insertions(+), 59 deletions(-) diff --git a/languages/python/python/stackencrypt/_declaration.py b/languages/python/python/stackencrypt/_declaration.py index aba90b7d5..ced4b7df1 100644 --- a/languages/python/python/stackencrypt/_declaration.py +++ b/languages/python/python/stackencrypt/_declaration.py @@ -8,10 +8,11 @@ The fields are the library's own list where one exists (dataclasses, attrs, Pydantic, ``NamedTuple``), and otherwise the class's type hints minus -``ClassVar`` and ``InitVar``, as on a SQLAlchemy declarative class. One generic -class around ``Annotated`` is read by structure, so ``Mapped[Annotated[str, -TextEq]]`` reads as ``TextEq``, with no SQLAlchemy import. A list, ``Final`` -and the other generics of the standard library are not read through. +``ClassVar``, ``InitVar`` and dunder names such as ``__tablename__``, as on a +SQLAlchemy declarative class. One generic class around ``Annotated`` is read +by structure, so ``Mapped[Annotated[str, TextEq]]`` reads as ``TextEq``, with +no SQLAlchemy import. A list, ``Final`` and the other generics of the +standard library are not read through. Then, in this order: @@ -80,6 +81,15 @@ class around ``Annotated`` is read by structure, so ``Mapped[Annotated[str, {"builtins", "collections", "collections.abc", "types", "typing"} ) +# The package whose wrappers, such as Mapped, are refused inside the marker's +# Annotated. A one-argument generic from any other package, such as +# re.Pattern[str], may be a value type. +_WRAPPER_PACKAGE = "sqlalchemy" + +# What evaluating a type hint raises when a name in it is missing or the +# string is not an expression. +_UNRESOLVED = (NameError, TypeError, AttributeError, SyntaxError) + # The one Python type of each marker kind. A value type must be exactly this # type: not Optional, not a union, and not a subclass. _KIND_TYPES: Dict[str, type] = { @@ -272,7 +282,7 @@ def _hints(cls: type) -> Dict[str, Any]: if not skip: try: return get_type_hints(cls, include_extras=True) - except (NameError, TypeError) as error: + except _UNRESOLVED as error: raise _hints_unresolved(cls, error) from error hints: Dict[str, Any] = {} # As get_type_hints does: each base's own hints in its own namespaces, @@ -280,24 +290,30 @@ def _hints(cls: type) -> Dict[str, Any]: for base in reversed(cls.__mro__): try: own = _own_annotations(base) - except (NameError, TypeError) as error: + except _UNRESOLVED as error: raise _hints_unresolved(cls, error) from error - module = sys.modules.get(base.__module__) - global_names = dict(vars(module)) if module is not None else {} for name, hint in own.items(): if name in skip: continue - - def probe() -> None: ... - - probe.__annotations__ = {name: hint} try: - hints.update(get_type_hints(probe, global_names, dict(vars(base)), include_extras=True)) - except (NameError, TypeError) as error: + hints[name] = _resolve_one(base, name, hint) + except _UNRESOLVED as error: raise _hints_unresolved(cls, error) from error return hints +def _resolve_one(base: type, name: str, hint: Any) -> Any: + """One hint of ``base``, resolved as ``get_type_hints`` resolves it. + + The probe is a class, not a function: a function's annotation cannot be + a ``ClassVar``, and before 3.14 a string ``ClassVar`` hint raises there. + """ + module = sys.modules.get(base.__module__) + global_names = dict(vars(module)) if module is not None else {} + probe = type("probe", (), {"__annotations__": {name: hint}}) + return get_type_hints(probe, global_names, dict(vars(base)), include_extras=True)[name] + + def _relationships(cls: type) -> frozenset: """The names of a SQLAlchemy class's relationships, such as ``books`` in ``books: Mapped[list["Book"]] = relationship()``. Empty for any other @@ -362,7 +378,9 @@ def _read_fields(cls: type, hints: Dict[str, Any]) -> List[_ReadField]: names = [ name for name, hint in hints.items() - if not _is_class_level(hint) and name not in relationships + if not _is_class_level(hint) + and not _is_dunder(name) + and name not in relationships ] read = [] for name in names: @@ -390,6 +408,12 @@ def _library_field_names(cls: type) -> Optional[List[str]]: return None +def _is_dunder(name: str) -> bool: + """A name such as ``__tablename__``: the class's configuration, which + the user cannot mark, not a field.""" + return len(name) > 4 and name.startswith("__") and name.endswith("__") + + def _is_class_level(hint: Any) -> bool: """A ``ClassVar`` or ``InitVar`` hint, which is not a field.""" if isinstance(hint, dataclasses.InitVar) or hint is dataclasses.InitVar: @@ -421,18 +445,12 @@ def _unresolved_field(cls: type) -> Optional[str]: for base in reversed(cls.__mro__): try: own = _own_annotations(base) - except (NameError, TypeError): + except _UNRESOLVED: return None - module = sys.modules.get(base.__module__) - global_names = dict(vars(module)) if module is not None else {} for name, hint in own.items(): - - def probe() -> None: ... - - probe.__annotations__ = {name: hint} try: - get_type_hints(probe, global_names, dict(vars(base)), include_extras=True) - except (NameError, TypeError): + _resolve_one(base, name, hint) + except _UNRESOLVED: return name return None @@ -478,6 +496,14 @@ def _is_wrapper(hint: Any) -> bool: ) +def _is_orm_wrapper(hint: Any) -> bool: + """A SQLAlchemy wrapper, such as ``Mapped[str]``, found by its module.""" + return ( + _is_wrapper(hint) + and get_origin(hint).__module__.partition(".")[0] == _WRAPPER_PACKAGE + ) + + def _without_none(value_type: Any) -> Tuple[Any, bool]: """``Optional[T]`` as ``(T, True)``; any other type as ``(type, False)``. ``Optional[T]`` is ``Union[T, None]``, and ``T | None`` from Python 3.10. @@ -494,9 +520,10 @@ def _marker_of(name: str, hint: Any, *, from_hints: bool) -> Tuple[Type[Marker], """The one marker of a field and its value type, or a ``DeclarationError``. ``from_hints`` says the field list came from the type hints, as on a - SQLAlchemy class. Only there is a wrapper inside the marker's + SQLAlchemy class. Only there is a SQLAlchemy wrapper inside the marker's ``Annotated`` refused as misplaced: ``Annotated[Mapped[str], TextEq]``. - Elsewhere a generic value type is a type mismatch, checked later. + Any other generic value type, such as ``re.Pattern[str]``, is the value's + own: a sealed marker refuses it later, as a type mismatch. """ if hint is None: raise _no_marker(name) @@ -512,7 +539,7 @@ def _marker_of(name: str, hint: Any, *, from_hints: bool) -> Tuple[Type[Marker], # Annotated outermost, with the wrapper inside it, is the order reversed. # Inside a wrapper already read, a generic value type is the value's own, # as in a relationship: Mapped[Annotated[list[Post], Ignore]]. - if from_hints and not unwrapped and markers and _is_wrapper(value_type): + if from_hints and not unwrapped and markers and _is_orm_wrapper(value_type): raise _marker_not_outermost(name, markers[0], value_type) if _nested_markers(hint): raise _marker_not_outermost(name, None, None) @@ -627,7 +654,8 @@ def _check_constructor(cls: type, read: List[_ReadField]) -> None: for field in read: parameter = parameters.get(field.name) if field.marker is Ignore: - if parameter is not None and _required(parameter): + taken_as = parameters.get(_init_name(cls, field.name)) + if taken_as is not None and _required(taken_as): raise DeclarationError( "declaration_ignore_required", f"field {field.name!r} is marked Ignore, but the constructor " @@ -660,20 +688,46 @@ def _takes( cls: type, name: str, parameter: Optional[inspect.Parameter], any_keyword: bool ) -> bool: """Whether the constructor takes field ``name`` as a keyword argument.""" - fields = getattr(cls, "__pydantic_fields__", None) - info = fields.get(name) if isinstance(fields, dict) else None + info = _pydantic_field(cls, name) if info is not None: # Pydantic's signature shows a field's alias, not its name. Its config # says whether the name is taken too. With extra="allow" the signature - # has **kwargs, and a name that is not taken lands in the extras. + # has **kwargs, and a name that is not taken lands in the extras. A + # Pydantic dataclass keeps its config in __pydantic_config__. alias = getattr(info, "validation_alias", None) or getattr(info, "alias", None) if alias is None or alias == name: return True - config = getattr(cls, "model_config", {}) + config = ( + getattr(cls, "model_config", None) or getattr(cls, "__pydantic_config__", None) or {} + ) return bool(config.get("populate_by_name") or config.get("validate_by_name")) return _keyword(parameter) or any_keyword +def _pydantic_field(cls: type, name: str) -> Any: + """Pydantic's record of field ``name``, or ``None``.""" + fields = getattr(cls, "__pydantic_fields__", None) + return fields.get(name) if isinstance(fields, dict) else None + + +def _init_name(cls: type, name: str) -> str: + """The parameter name the constructor gives field ``name``. + + Pydantic names it by the field's alias. attrs names a private field + without its leading underscore, so ``_cache`` is ``cache``. + """ + info = _pydantic_field(cls, name) + if info is not None: + for alias in (getattr(info, "validation_alias", None), getattr(info, "alias", None)): + if isinstance(alias, str): + return alias + return name + for attribute in getattr(cls, "__attrs_attrs__", ()): + if attribute.name == name: + return getattr(attribute, "alias", None) or name + return name + + def _keyword(parameter: Optional[inspect.Parameter]) -> bool: return parameter is not None and parameter.kind in ( inspect.Parameter.POSITIONAL_OR_KEYWORD, diff --git a/languages/python/python/stackencrypt/_redact.py b/languages/python/python/stackencrypt/_redact.py index 2c60ad0bb..e733d9dbe 100644 --- a/languages/python/python/stackencrypt/_redact.py +++ b/languages/python/python/stackencrypt/_redact.py @@ -7,17 +7,17 @@ Pydantic's ``model_dump``, ``vars`` and ``__dict__`` still return plaintext, and so does a debugger that reads attributes. -A generated method (dataclass, attrs, Pydantic, ``NamedTuple``) is replaced. -A method written on the class itself is refused unless the decorator is +A method that dataclasses, attrs or ``NamedTuple`` generated is replaced. +Pydantic's methods are on its base classes. Anything else on the class +itself is the user's: a function, a ``staticmethod``, a ``partialmethod``, +or a function from another module. It is refused unless the decorator is given ``redact=False``, because replacing it would silently drop the user's code. A method inherited from a base is replaced on the declared class only. """ import collections -import dataclasses import inspect import os -import reprlib import types from typing import Any, Callable, Iterator, Sequence, Tuple @@ -45,38 +45,53 @@ def __repr__(self) -> str: type(vars(collections.namedtuple("_Probe", "field"))["field"]), ) -# The files a generated __repr__ reports as its own. A dataclass's __repr__ is -# wrapped for recursion, and the wrapper is in reprlib.py on 3.14 and in -# dataclasses.py on 3.9. attrs reports a "" name. -_GENERATOR_FILES = frozenset( - os.path.realpath(module.__file__) - for module in (dataclasses, reprlib) - if module.__file__ is not None -) - # attrs compiles its methods under this file name. _ATTRS_GENERATED = " bool: - """Whether ``cls`` itself has a ``name`` method written by hand. + """Whether ``cls`` itself has a ``name`` that no known generator wrote. Only the class's own ``__dict__`` is read: a base's method is not the - class's. A generated method comes from a generator's file, or from - another module, such as ``pydantic.main``. A method compiled from a - string, as in the REPL, a notebook or ``python -c``, is hand-written. + class's. Anything there is the user's unless it is a generator's output. + A method compiled from a string, as in the REPL, a notebook or + ``python -c``, is hand-written. """ - method = vars(cls).get(name) - code = getattr(method, "__code__", None) - if code is None or getattr(method, "__module__", None) != cls.__module__: + found = vars(cls) + return name in found and not _generated(found[name]) + + +def _generated(method: object) -> bool: + """Whether dataclasses, attrs or ``NamedTuple`` generated ``method``. + + A wrapper, such as the one ``reprlib.recursive_repr`` adds, is + unwrapped first, so the function inside it decides. A user's + ``recursive_repr`` is then the user's own function. + """ + if not isinstance(method, types.FunctionType): + return False + inner = inspect.unwrap(method) + code = getattr(inner, "__code__", None) + if code is None: return False filename = code.co_filename - return ( - not filename.startswith(_ATTRS_GENERATED) - and os.path.realpath(filename) not in _GENERATOR_FILES - ) + if filename.startswith(_ATTRS_GENERATED): + return True + if filename == _DATACLASS_FILE: + return inner.__qualname__.startswith(_DATACLASS_QUALNAME) + return _COLLECTIONS_FILE is not None and os.path.realpath(filename) == _COLLECTIONS_FILE def install(cls: type, fields: Sequence[Tuple[str, bool]]) -> Tuple[Callable[..., Any], ...]: diff --git a/languages/python/tests/_declaration_cases.py b/languages/python/tests/_declaration_cases.py index 5992001e7..cc84ce540 100644 --- a/languages/python/tests/_declaration_cases.py +++ b/languages/python/tests/_declaration_cases.py @@ -10,12 +10,14 @@ import datetime import decimal import pickle +import re import sys import uuid from typing import Annotated, ClassVar, Final, List, NamedTuple, Optional, Union import attrs import pydantic +import pydantic.dataclasses import pytest from sqlalchemy import inspect from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship @@ -236,6 +238,29 @@ class User(KeywordConstructor): assert planned(User) == [("email", TextEq)] +def test_a_generic_value_type_from_another_module_is_not_a_misplaced_wrapper(): + # re.Pattern[str] is a one-argument generic, but it is a value type, not + # a wrapper such as Mapped. + @stackencrypt.model("users") + class User(KeywordConstructor): + email: Annotated[str, TextEq] + pattern: Annotated[re.Pattern[str], Ignore] + + assert planned(User) == [("email", TextEq)] + + +def test_a_dunder_annotation_is_not_a_field(): + # As on a SQLAlchemy class: __tablename__ is configuration, not a column, + # and the user cannot mark it. + @stackencrypt.model("users") + class User(KeywordConstructor): + __tablename__: str = "users" + __table_args__: tuple = () + email: Annotated[str, TextEq] + + assert planned(User) == [("email", TextEq)] + + @pytest.mark.parametrize("kind", ["dataclass", "plain"]) def test_a_marker_inside_a_list_or_final_is_refused(kind): def build_list(): @@ -305,6 +330,66 @@ class User: assert isinstance(error.__cause__, NameError) +def build_with_a_bad_hint(kind): + if kind == "missing attribute": + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + posts: Annotated[List["sys.Undefined"], Ignore] = dataclasses.field( + default_factory=list + ) + + else: + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + posts: "List[" = dataclasses.field(default_factory=list) # noqa: F722 + + +@pytest.mark.parametrize( + "kind, cause", + [("missing attribute", AttributeError), ("malformed string", SyntaxError)], +) +def test_a_hint_that_fails_to_evaluate_is_refused_with_its_field(kind, cause): + error = refusal(lambda: build_with_a_bad_hint(kind)) + assert error.code == "declaration_hints_unresolved" + assert error.field == "posts" + assert isinstance(error.__cause__, cause) + + +def build_with_a_class_var_before_an_unresolved_hint(kind): + # A ClassVar hint written as a string resolves on a class, not as a + # function argument. Pydantic's own bases hold such hints. + if kind == "dataclass": + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + count: "ClassVar[int]" = 0 + email: Annotated[str, TextEq] = "" + posts: Annotated[List["Undefined"], Ignore] = dataclasses.field( # noqa: F821 + default_factory=list + ) + + else: + + @stackencrypt.model("users") + class User(pydantic.BaseModel): + email: Annotated[str, TextEq] + posts: Annotated[List["Undefined"], Ignore] = [] # noqa: F821 + + +@pytest.mark.parametrize("kind", ["dataclass", "pydantic"]) +def test_an_unresolved_hint_after_a_class_var_is_named(kind): + error = refusal(lambda: build_with_a_class_var_before_an_unresolved_hint(kind)) + assert error.code == "declaration_hints_unresolved" + assert error.field == "posts" + + class LaterBase(DeclarativeBase): pass @@ -615,6 +700,13 @@ class User: class User: _email: Annotated[str, TextEq] + elif kind == "pydantic dataclass init=False field": + + @stackencrypt.model("users") + @pydantic.dataclasses.dataclass + class User: + email: Annotated[str, TextEq] = dataclasses.field(init=False) + elif kind == "pydantic alias": @stackencrypt.model("users") @@ -635,6 +727,7 @@ class User(pydantic.BaseModel): "init=False field", "init=False class", "attrs private name", + "pydantic dataclass init=False field", "pydantic alias", "pydantic alias with extra=allow", ], @@ -654,6 +747,16 @@ class User(pydantic.BaseModel): assert planned(User) == [("email", TextEq)] +def test_a_pydantic_dataclass_alias_is_accepted_when_its_config_takes_the_name(): + # A Pydantic dataclass keeps its config in __pydantic_config__. + @stackencrypt.model("users") + @pydantic.dataclasses.dataclass(config=pydantic.ConfigDict(populate_by_name=True)) + class User: + email: Annotated[str, TextEq] = pydantic.Field(alias="Email") + + assert planned(User) == [("email", TextEq)] + + def build_with_a_required_ignore(kind): if kind == "dataclass": @@ -671,6 +774,21 @@ class User: email: Annotated[str, TextEq] note: Annotated[str, Ignore] + elif kind == "attrs private name": + + @stackencrypt.model("users") + @attrs.define + class User: + email: Annotated[str, TextEq] + _note: Annotated[str, Ignore] + + elif kind == "pydantic alias": + + @stackencrypt.model("users") + class User(pydantic.BaseModel): + email: Annotated[str, TextEq] + note: Annotated[str, Ignore] = pydantic.Field(alias="Note") + else: @stackencrypt.model("users") @@ -679,11 +797,20 @@ class User(pydantic.BaseModel): note: Annotated[str, Ignore] -@pytest.mark.parametrize("kind", ["dataclass", "attrs", "pydantic"]) -def test_a_required_ignore_field_is_refused(kind): +@pytest.mark.parametrize( + "kind, field", + [ + ("dataclass", "note"), + ("attrs", "note"), + ("attrs private name", "_note"), + ("pydantic", "note"), + ("pydantic alias", "note"), + ], +) +def test_a_required_ignore_field_is_refused(kind, field): error = refusal(lambda: build_with_a_required_ignore(kind)) assert error.code == "declaration_ignore_required" - assert error.field == "note" + assert error.field == field assert "default" in str(error) diff --git a/languages/python/tests/test_redaction.py b/languages/python/tests/test_redaction.py index c47cc67ac..0bc5ed87d 100644 --- a/languages/python/tests/test_redaction.py +++ b/languages/python/tests/test_redaction.py @@ -1,6 +1,8 @@ """``repr`` and ``str`` of a declared class show no sealed value.""" import dataclasses +import functools +import reprlib from typing import Annotated, NamedTuple import attrs @@ -198,6 +200,60 @@ def __init__(self, **kwargs): assert repr(User(email=SECRET, age=9)) == "User(email=, age=9)" +def _repr_from_another_module(self): + return "elsewhere" + + +_repr_from_another_module.__module__ = "somewhere.else" + + +def build_with_a_hand_written_method(form): + if form == "recursive_repr": + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + + @reprlib.recursive_repr() + def __repr__(self): + return f"User({self.email})" + + elif form == "staticmethod": + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + __str__ = staticmethod(lambda: "user") + + elif form == "partialmethod": + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + __str__ = functools.partialmethod(lambda self, text: text, "user") + + else: + + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + __repr__ = _repr_from_another_module + + +@pytest.mark.parametrize( + "form", ["recursive_repr", "staticmethod", "partialmethod", "another module"] +) +def test_a_hand_written_method_in_any_form_is_refused(form): + # Only a generator's own output is replaced. Anything else written on the + # class is the user's, and replacing it would drop their code silently. + error = pytest.raises(DeclarationError, build_with_a_hand_written_method, form).value + assert error.code == "declaration_repr_present" + + def test_a_hand_written_str_is_refused(): def build(): @stackencrypt.model("users") From 20a82b60297572af385eb5d9774b4ec078aa3976 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 15:12:19 +0900 Subject: [PATCH 03/21] fix(python): give encrypt and decrypt the right error for each bad input encrypt read each field by name, so a model of another class with the same field names was encrypted under this table. It now refuses a model that is not an instance of the declared class, with value_wrong_type. A datetime whose tzinfo gives no UTC offset is naive in Python. The value reader now checks utcoffset(), so such a value gives value_naive_datetime, not a TypeError. decrypt turned every error from json.dumps into decrypt_malformed. That included KeyboardInterrupt and MemoryError. Now only TypeError, ValueError, and RecursionError do so. An error raised while a passthrough value is read now propagates, as it does on encrypt. One row passed on its own, for example from fetchone(), was read as a list of column names. decrypt, decrypt_rows, and encrypt_rows now refuse it and say to pass [row]. A str with a lone surrogate now gives decrypt_malformed, the same as bytes that are not UTF-8. encrypt now has the type Sequence[T]. Addresses review threads 1499-F5, 1499-F6, and 1509-F4, F5, F6, F8, F9. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/crates/engine/src/decrypt.rs | 50 +++++++++----- languages/python/crates/engine/src/values.rs | 6 +- .../python/python/stackencrypt/_cipher.py | 41 ++++++++++-- languages/python/tests/test_decrypt.py | 65 +++++++++++++++++++ languages/python/tests/test_decrypt_rows.py | 6 ++ languages/python/tests/test_encrypt.py | 23 +++++++ languages/python/tests/test_encrypt_rows.py | 6 ++ 7 files changed, 171 insertions(+), 26 deletions(-) diff --git a/languages/python/crates/engine/src/decrypt.rs b/languages/python/crates/engine/src/decrypt.rs index 0ab1fde14..9eb8c1036 100644 --- a/languages/python/crates/engine/src/decrypt.rs +++ b/languages/python/crates/engine/src/decrypt.rs @@ -9,6 +9,7 @@ //! only after every value has been turned into a Python object, so a call //! that fails returns no instance. +use pyo3::exceptions::{PyRecursionError, PyTypeError, PyUnicodeEncodeError, PyValueError}; use pyo3::prelude::*; use pyo3::sync::PyOnceLock; use pyo3::types::{PyBytes, PyDict, PyList, PyString}; @@ -130,7 +131,7 @@ fn stored_row(plan: &Plan, row: &Bound<'_, PyAny>, at: usize) -> PyResult { .eql(&plan.plan, field, text.as_bytes()) .map_err(in_row)?; } - None => passthrough(&mut stored, field, *kind, &value).map_err(in_row)?, + None => passthrough(&mut stored, field, *kind, &value)?.map_err(in_row)?, } } Ok(stored) @@ -146,7 +147,10 @@ fn stored_row(plan: &Plan, row: &Bound<'_, PyAny>, at: usize) -> PyResult { /// from 3.10. The text is then copied once more, into the engine's frame. /// /// The outer error is Python's own; the inner one is the refusal, for the -/// caller to name the row in. +/// caller to name the row in. Only an error that is about the value +/// becomes a refusal: a `str` with no UTF-8 encoding, or a `dict` that +/// `json.dumps` refuses. Any other error, such as `MemoryError` or +/// `KeyboardInterrupt`, propagates as it is. fn eql_text<'py>( field: &Field, value: &Bound<'py, PyAny>, @@ -161,7 +165,14 @@ fn eql_text<'py>( })) }; if let Ok(text) = value.cast::() { - return Ok(Ok(text.encode_utf8()?)); + // A lone surrogate is a str Python holds with no UTF-8 encoding. + return match text.encode_utf8() { + Ok(bytes) => Ok(Ok(bytes)), + Err(error) if error.is_instance_of::(py) => { + malformed("is a str that is not valid UTF-8".to_owned()) + } + Err(error) => Err(error), + }; } if let Ok(bytes) = value.cast::() { return match std::str::from_utf8(bytes.as_bytes()) { @@ -172,7 +183,14 @@ fn eql_text<'py>( if value.cast::().is_ok() { return match dumps(py)?.call1((value,)) { Ok(text) => Ok(Ok(text.cast::()?.encode_utf8()?)), - Err(_) => malformed("is a dict that is not JSON".to_owned()), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + malformed("is a dict that is not JSON".to_owned()) + } + Err(error) => Err(error), }; } if value.is_none() { @@ -185,38 +203,34 @@ fn eql_text<'py>( } /// A passthrough column's value, checked by exact type as on `encrypt`. +/// +/// An error Python raises while the value is read, such as one from a +/// user's `tzinfo`, propagates as it is, as on `encrypt`. The inner error +/// is the refusal. fn passthrough( stored: &mut Row, field: &Field, kind: PyKind, value: &Bound<'_, PyAny>, -) -> Result<(), Refusal> { +) -> PyResult> { let named = |refused: Refusal| Refusal { message: format!("field {:?}: {}", field.name, refused.message), field: Some(field.name.clone()), reason: Some("passthrough".to_owned()), ..refused }; - let read = match read(value, kind, "passthrough") { - Ok(Ok(read)) => read, - Ok(Err(refused)) => return Err(named(refused)), - Err(error) => { - return Err(named(Refusal { - code: "decrypt_malformed".to_owned(), - message: format!("the value could not be read: {error}"), - field: None, - reason: None, - })) - } + let read = match read(value, kind, "passthrough")? { + Ok(read) => read, + Err(refused) => return Ok(Err(named(refused))), }; - stored.passthrough(field, read.value()).map_err(|misfit| { + Ok(stored.passthrough(field, read.value()).map_err(|misfit| { named(Refusal { code: misfit.code().to_owned(), message: "the value is outside what the engine stores.".to_owned(), field: None, reason: None, }) - }) + })) } /// An opened value as the Python object of its field. diff --git a/languages/python/crates/engine/src/values.rs b/languages/python/crates/engine/src/values.rs index 4d24fec7e..524152e11 100644 --- a/languages/python/crates/engine/src/values.rs +++ b/languages/python/crates/engine/src/values.rs @@ -153,7 +153,7 @@ impl Read<'_> { /// # Errors /// /// `value_none` for `None`, `value_wrong_type` for another type, -/// `value_naive_datetime` for a datetime with no `tzinfo`, and the range +/// `value_naive_datetime` for a datetime with no UTC offset, and the range /// refusals for an `int` or `Decimal` the engine cannot hold. `field` is /// not set: the caller names it. pub fn read<'a>( @@ -197,7 +197,9 @@ pub fn read<'a>( other(Value::Date(ordinal - 719_163)) } PyKind::DateTime => { - if value.getattr("tzinfo")?.is_none() { + // Naive as Python defines it: no tzinfo, or a tzinfo whose + // utcoffset() is None. The subtraction below needs an offset. + if value.call_method0("utcoffset")?.is_none() { return Ok(Err(refusal( "value_naive_datetime", "is a naive datetime. Give it a tzinfo, such as timezone.utc", diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index 000f6069b..bebc3415d 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -73,7 +73,7 @@ def __init__(self, cls: Type[T]) -> None: # threads may build the same plan at once; either one is kept. self._row_plans: Dict[FrozenSet[str], object] = {} - def encrypt(self, models: Sequence[object]) -> List[Dict[str, object]]: + def encrypt(self, models: Sequence[T]) -> List[Dict[str, object]]: """Encrypt ``models``, instances of the declared class. Returns one dict per model, from column name to column value: the @@ -83,9 +83,23 @@ def encrypt(self, models: Sequence[object]) -> List[Dict[str, object]]: Every value is checked first, by exact type and range. A value that does not fit raises ``InvalidValueError`` naming the field, and no - request is made. An empty list makes no request at all. + request is made. A model that is not an instance of the declared + class raises ``InvalidValueError`` with the code + ``value_wrong_type``. An empty list makes no request at all. """ - return _engine.encrypt(self._plan, list(models)) + models = list(models) + for at, model in enumerate(models): + # The engine reads each field by name, so a model of another + # class with the same field names would be encrypted under this + # table. + if not isinstance(model, self._cls): + raise InvalidValueError( + "value_wrong_type", + f"model {at} is a {type(model).__name__}, not a " + f"{self._cls.__name__}. Encrypt only instances of the " + "declared class.", + ) + return _engine.encrypt(self._plan, models) def encrypt_rows(self, rows: Sequence[Mapping[str, object]]) -> List[Dict[str, object]]: """Encrypt ``rows``, mappings from field name to value, that may @@ -108,7 +122,7 @@ def encrypt_rows(self, rows: Sequence[Mapping[str, object]]) -> List[Dict[str, o raises ``InvalidValueError`` with the code ``value_row_shape``. No request is made when any row fails. The input rows are not changed. """ - rows = list(rows) + rows = _rows(rows, InvalidValueError, "value_row_shape") names = self._row_fields(rows, InvalidValueError, "value_row_shape", "Encrypt") if not names: return [dict(row) for row in rows] @@ -138,7 +152,7 @@ def decrypt_rows(self, rows: Sequence[Mapping[str, object]]) -> List[Dict[str, o fields than row 0, raises ``DecryptError`` with the code ``decrypt_row_shape``. No request is made when any row fails. """ - rows = list(rows) + rows = _rows(rows, DecryptError, "decrypt_row_shape") names = self._row_fields(rows, DecryptError, "decrypt_row_shape", "Decrypt") if not names: return [dict(row) for row in rows] @@ -197,7 +211,9 @@ def decrypt(self, rows: Sequence[Mapping[str, object]]) -> List[T]: field's raises ``InvalidValueError``; it is never converted. The call is all or nothing: no instance is built when any row fails. """ - return _engine.decrypt(self._plan, self._cls, list(rows)) + return _engine.decrypt( + self._plan, self._cls, _rows(rows, DecryptError, "decrypt_row_shape") + ) def eq(self, field: str, value: object) -> str: """The equality query term for ``value`` on the declared field @@ -399,6 +415,19 @@ def _not_decorated(cls: object) -> DeclarationError: ) +def _rows(rows: object, error: Type[StashError], code: str) -> list: + """``rows`` as a list. Raises ``error`` with ``code`` for one row on its + own, such as ``cursor.fetchone()`` returns: ``list()`` of a mapping is its + keys, and each key would be refused as a row that is not a mapping.""" + if isinstance(rows, Mapping): + raise error( + code, + "this takes a list of rows, not one row on its own. Pass [row], " + "or the result of fetchall().", + ) + return list(rows) # type: ignore[call-overload] + + def _build_plan(declaration: Declaration, fields: Sequence[DeclaredField]) -> object: """The engine's plan for ``fields`` of ``declaration``, in their order.""" return _engine.build_plan( diff --git a/languages/python/tests/test_decrypt.py b/languages/python/tests/test_decrypt.py index 9a4c94768..6a01f03f3 100644 --- a/languages/python/tests/test_decrypt.py +++ b/languages/python/tests/test_decrypt.py @@ -7,6 +7,7 @@ """ import dataclasses +import datetime import json from typing import Annotated @@ -57,6 +58,70 @@ def test_a_tuple_row_is_the_wrong_shape_and_names_dict_row(untouched): assert "dict_row" in error.message +def test_one_row_on_its_own_is_refused_as_one_row(untouched): + # As from cursor.fetchone(): list() of a dict is its column names. + error = refused({"email": STORED, "age": 3}) + assert error.code == "decrypt_row_shape" + assert "not one row" in error.message + + +def test_a_str_that_is_not_valid_utf8_is_malformed(untouched): + # A lone surrogate: a str Python holds, with no UTF-8 encoding. + error = refused([{"email": "\ud800", "age": 3}]) + assert error.code == "decrypt_malformed" + assert error.field == "email" + + +class Boom(Exception): + """An error that is not about the input's JSON or type.""" + + +class ExplodingDict(dict): + """A dict whose items, which json.dumps reads, raise Boom.""" + + def items(self): + raise Boom("not about the JSON") + + +def test_an_error_that_is_not_about_the_json_propagates(untouched): + with pytest.raises(Boom): + Cipher(User).decrypt([{"email": ExplodingDict(i={}), "age": 3}]) + + +class NoOffset(datetime.tzinfo): + def utcoffset(self, dt): + return None + + +class RaisingOffset(datetime.tzinfo): + def utcoffset(self, dt): + raise Boom("the user's own tzinfo") + + +@stackencrypt.model("visits") +@dataclasses.dataclass +class Visit: + email: Annotated[str, TextEq] + seen: Annotated[datetime.datetime, Passthrough] + + +VISIT_STORED = json.dumps({"i": {"t": "visits", "c": "email"}, "v": 2}) + + +def test_a_passthrough_datetime_whose_tzinfo_gives_no_offset_is_naive(untouched): + seen = datetime.datetime(2026, 1, 1, tzinfo=NoOffset()) + with pytest.raises(InvalidValueError) as raised: + Cipher(Visit).decrypt([{"email": VISIT_STORED, "seen": seen}]) + assert raised.value.code == "value_naive_datetime" + assert raised.value.field == "seen" + + +def test_an_error_from_a_passthrough_value_propagates(untouched): + seen = datetime.datetime(2026, 1, 1, tzinfo=RaisingOffset()) + with pytest.raises(Boom): + Cipher(Visit).decrypt([{"email": VISIT_STORED, "seen": seen}]) + + def test_a_row_without_a_declared_column_names_it(untouched): error = refused([{"email": STORED}]) assert error.code == "decrypt_missing_column" diff --git a/languages/python/tests/test_decrypt_rows.py b/languages/python/tests/test_decrypt_rows.py index be545bcb3..170f43d90 100644 --- a/languages/python/tests/test_decrypt_rows.py +++ b/languages/python/tests/test_decrypt_rows.py @@ -58,6 +58,12 @@ def test_rows_with_no_declared_field_come_back_unchanged_with_no_request(people, assert people.decrypt_rows(rows) == rows +def test_one_row_on_its_own_is_refused_as_one_row(people, untouched): + error = refused(people, {"email": STORED, "age": 3}) + assert error.code == "decrypt_row_shape" + assert "not one row" in error.message + + def test_a_row_that_is_not_a_mapping_is_refused(people, untouched): error = refused(people, [{"email": STORED}, (STORED,)]) assert error.code == "decrypt_row_shape" diff --git a/languages/python/tests/test_encrypt.py b/languages/python/tests/test_encrypt.py index 305a92445..7e5ff5cb6 100644 --- a/languages/python/tests/test_encrypt.py +++ b/languages/python/tests/test_encrypt.py @@ -149,6 +149,29 @@ def test_a_naive_datetime_is_refused(untouched): assert "tzinfo" in error.message +class NoOffset(datetime.tzinfo): + """A tzinfo that gives no offset: Python counts its datetime as naive.""" + + def utcoffset(self, dt): + return None + + +def test_a_datetime_whose_tzinfo_gives_no_offset_is_naive(untouched): + cipher, User = cipher_for(seen=Annotated[datetime.datetime, Timestamp]) + seen = datetime.datetime(2026, 1, 1, tzinfo=NoOffset()) + error = refused(cipher, User(email=SECRET, seen=seen)) + assert error.code == "value_naive_datetime" + + +def test_a_model_of_another_class_is_the_wrong_type(untouched): + # Contact has the same field names, so every getattr would succeed. + cipher, _ = cipher_for(age=Annotated[int, Passthrough]) + Contact = dataclasses.make_dataclass("Contact", [("email", str), ("age", int)]) + error = refused(cipher, Contact(email=SECRET, age=3)) + assert error.code == "value_wrong_type" + assert "model 0 is a Contact" in error.message + + def test_every_model_is_checked_before_any_request(untouched): # The bad value is in the last model of many. cipher, User = cipher_for(age=Annotated[int, Bigint]) diff --git a/languages/python/tests/test_encrypt_rows.py b/languages/python/tests/test_encrypt_rows.py index e4d5a7ca9..d76a29258 100644 --- a/languages/python/tests/test_encrypt_rows.py +++ b/languages/python/tests/test_encrypt_rows.py @@ -47,6 +47,12 @@ def test_rows_with_no_declared_field_come_back_unchanged_with_no_request(people, assert people.encrypt_rows(rows) == rows +def test_one_row_on_its_own_is_refused_as_one_row(people, untouched): + error = refused(people, {"email": SECRET}) + assert error.code == "value_row_shape" + assert "not one row" in error.message + + def test_a_row_that_is_not_a_mapping_is_refused(people, untouched): error = refused(people, [{"email": SECRET}, ("email", SECRET)]) assert error.code == "value_row_shape" From 0f9a4d3000e66810650238e9fa3b4e738d2a38ad Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 15:30:06 +0900 Subject: [PATCH 04/21] test(python): cover the paths the review found without a test Reviewers marked ten paths as having no test. Each one could break with no test failing. - The engine commit is checked against git: the SHA must equal HEAD, with -dirty exactly when a stack-* crate has changed outside fuzz/. build.rs now records where the commit came from. cargo sets the commit variable when it runs the tests, so the test cannot read the environment to find an override. - The frame reader walks a typed null, a sequence, and a nested map. A frame writer writes a typed null as the plain null, so the test expects no type there. - The SIGINT gate tests cancel and join their timer in finally. - A class read from its hints leaves out ClassVar and InitVar, and rule A holds inside Mapped on a real mapped class. - repr of an expired or detached SQLAlchemy row sends no SQL statement. - After the switch to the deterministic client, the live client and a second switch are refused. test:cargo runs this test binary with the deterministic-kms feature. A seed that is not 32 bytes is refused. - decrypt runs from 8 threads at once, live and deterministic. - The keyset test fails if the load-keyset counter is missing. Addresses review threads 1405-F2, 1414-F4, F6, 1433-F6, F7, 1453-F7, 1513-F2, F3, and 1516-F1, F2. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/crates/core/build.rs | 10 ++-- languages/python/crates/core/src/lib.rs | 47 ++++++++++++++++++ .../crates/core/tests/deterministic_switch.rs | 25 ++++++++++ .../crates/core/tests/gate_frame_reader.rs | 48 +++++++++++++++++-- languages/python/mise.toml | 3 ++ languages/python/tests/_declaration_cases.py | 45 +++++++++++++++++ languages/python/tests/_sync_contract.py | 26 ++++++++++ .../python/tests/test_concurrency_live.py | 16 ++++++- languages/python/tests/test_encrypt_live.py | 2 + .../tests/test_gate_keyboard_interrupt.py | 24 ++++++---- languages/python/tests/test_redaction.py | 23 ++++++++- .../tests_deterministic/test_concurrency.py | 16 ++++++- .../python/tests_deterministic/test_switch.py | 22 +++++++++ 13 files changed, 290 insertions(+), 17 deletions(-) create mode 100644 languages/python/crates/core/tests/deterministic_switch.rs create mode 100644 languages/python/tests_deterministic/test_switch.py diff --git a/languages/python/crates/core/build.rs b/languages/python/crates/core/build.rs index aadc701ed..80d873f9a 100644 --- a/languages/python/crates/core/build.rs +++ b/languages/python/crates/core/build.rs @@ -29,9 +29,9 @@ const ENGINE_PREFIX: &str = "stack-"; fn main() { println!("cargo:rerun-if-env-changed={ENV_OVERRIDE}"); - let commit = match env::var(ENV_OVERRIDE) { - Ok(value) if !value.is_empty() => value.to_ascii_lowercase(), - _ => commit_from_git(), + let (commit, source) = match env::var(ENV_OVERRIDE) { + Ok(value) if !value.is_empty() => (value.to_ascii_lowercase(), "env"), + _ => (commit_from_git(), "git"), }; let sha = commit.strip_suffix("-dirty").unwrap_or(&commit); assert!( @@ -39,6 +39,10 @@ fn main() { "the engine commit must be a full 40-character hex SHA (optionally `-dirty`), got {commit:?}" ); println!("cargo:rustc-env={ENV_OVERRIDE}={commit}"); + // Where the commit came from, for the test that checks it against git. + // cargo also sets the variable above when it runs the tests, so the + // test cannot tell an override from the environment. + println!("cargo:rustc-env=STACKENCRYPT_ENGINE_COMMIT_SOURCE={source}"); } fn commit_from_git() -> String { diff --git a/languages/python/crates/core/src/lib.rs b/languages/python/crates/core/src/lib.rs index 2d1981b16..98a165c56 100644 --- a/languages/python/crates/core/src/lib.rs +++ b/languages/python/crates/core/src/lib.rs @@ -57,6 +57,7 @@ pub fn engine_commit() -> &'static str { #[cfg(test)] mod tests { use super::engine_commit; + use std::process::Command; #[test] fn engine_commit_is_a_full_hex_sha() { @@ -68,4 +69,50 @@ mod tests { "{commit:?}" ); } + + /// `git` in this crate's directory, or `None` when it fails. + fn git(args: &[&str]) -> Option { + let output = Command::new("git") + .args(args) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output() + .ok()?; + output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).into_owned()) + } + + /// The commit is this checkout's, with `-dirty` exactly when a file + /// under a `packages/stack-*` crate has changed outside its `fuzz/`. + /// The status is read for all of `packages/` and filtered here, so the + /// test does not repeat build.rs's pathspec. + #[test] + fn engine_commit_is_this_checkouts_commit() { + // A commit given in STACKENCRYPT_ENGINE_COMMIT is trusted as given. + if env!("STACKENCRYPT_ENGINE_COMMIT_SOURCE") != "git" { + return; + } + let Some(head) = git(&["rev-parse", "HEAD"]) else { + return; + }; + let status = git(&["status", "--porcelain", "--", "../../../../packages"]) + .expect("git status runs in a checkout"); + // A porcelain line is a two-letter status, a space and the path from + // the repository root. A rename shows "old -> new". + let dirty = status.lines().any(|line| { + line.get(3..).is_some_and(|paths| { + paths.split(" -> ").any(|path| { + path.trim_matches('"') + .strip_prefix("packages/") + .and_then(|rest| rest.split_once('/')) + .is_some_and(|(krate, rest)| { + krate.starts_with("stack-") && !rest.starts_with("fuzz/") + }) + }) + }) + }); + let expected = format!("{}{}", head.trim(), if dirty { "-dirty" } else { "" }); + assert_eq!(engine_commit(), expected, "git status:\n{status}"); + } } diff --git a/languages/python/crates/core/tests/deterministic_switch.rs b/languages/python/crates/core/tests/deterministic_switch.rs new file mode 100644 index 000000000..3809a4ea2 --- /dev/null +++ b/languages/python/crates/core/tests/deterministic_switch.rs @@ -0,0 +1,25 @@ +//! The switch to the deterministic client of the `deterministic-kms` test +//! build. A test binary is its own process, and the switch is for the whole +//! process, so these checks run in this binary alone. `test:cargo` runs it +//! with the feature on. + +#![cfg(feature = "deterministic-kms")] + +use stackencrypt_core::client; + +#[test] +fn after_the_switch_the_live_client_is_refused_and_a_second_switch_too() { + client::use_deterministic([7; 32]).expect("the first switch succeeds"); + assert!(client::deterministic().is_some()); + + // An operation that skipped the check for the deterministic client + // would send requests to ZeroKMS. It is refused instead. + let Err(refused) = client::shared() else { + panic!("the live client is refused"); + }; + assert_eq!(refused.code, "stack_test_build"); + assert_eq!(client::build_count(), 0); + + let refused = client::use_deterministic([7; 32]).expect_err("a second switch is refused"); + assert_eq!(refused.code, "stack_test_build"); +} diff --git a/languages/python/crates/core/tests/gate_frame_reader.rs b/languages/python/crates/core/tests/gate_frame_reader.rs index 9d1dbd42e..d444b47a1 100644 --- a/languages/python/crates/core/tests/gate_frame_reader.rs +++ b/languages/python/crates/core/tests/gate_frame_reader.rs @@ -8,7 +8,7 @@ use stack_encrypt::dynamic::frame::Frame; use stack_encrypt::dynamic::plaintext::{ - Encrypt, Error, FixedType, Int, Logical, MapSink, Physical, SeqSink, Sink, SourceError, + Encrypt, Error, FixedType, Int, Logical, MapSink, Physical, SeqSink, Sink, SourceError, Width, }; use stack_encrypt::dynamic::ValueFrame; @@ -30,6 +30,28 @@ impl Encrypt for User { } } +/// A record with the other shapes a field can have: a typed null, a +/// sequence and a nested map. The typed null is written as a plain null. +struct Shapes; + +impl Encrypt for Shapes { + fn visit(self, sink: S) -> Result { + let mut map = sink.map(Some(3))?; + let date = FixedType::new(Some(Logical::Date), Width::I32).expect("a date has a width"); + map.entry("left")?.typed_null(date)?; + let mut tags = map.entry("tags")?.seq(Some(2))?; + tags.element()?.scalar(None, Physical::Text("a"))?; + tags.element()?.scalar(None, Physical::Int(Int::I64(2)))?; + tags.end()?; + let mut address = map.entry("address")?.map(Some(1))?; + address + .entry("city")? + .scalar(None, Physical::Text("London"))?; + address.end()?; + map.end() + } +} + #[derive(Clone)] struct Nickname; @@ -92,8 +114,8 @@ impl Sink for Reader<'_> { Ok(()) } - fn typed_null(self, _: FixedType) -> Result<(), ReadError> { - self.scalar(None, Physical::Null) + fn typed_null(self, ty: FixedType) -> Result<(), ReadError> { + self.scalar(ty.logical(), Physical::Null) } fn seq(self, _: Option) -> Result { @@ -192,6 +214,26 @@ fn an_outside_sink_recovers_each_field_kind_from_a_record_frame() { ); } +#[test] +fn an_outside_sink_reads_a_typed_null_a_sequence_and_a_nested_map() { + let record = ValueFrame::of(Shapes).expect("the record is in the value model"); + let frame = Frame::parse(record.risky_ref()).expect("a ValueFrame is one frame"); + + assert_eq!( + read(frame), + vec![ + // A frame writer writes a typed null as the plain null: a + // frame is plaintext, so padding hides nothing in it (vitaminc's + // FrameWriter). A decrypted record's null carries no type, and + // the extension takes a field's type from its plan. + seen("left", None, "null", false), + seen("tags", None, "text", false), + seen("tags", None, "int", false), + seen("city", None, "text", false), + ], + ); +} + #[test] fn malformed_frame_bytes_are_refused_before_any_walk() { let record = ValueFrame::of(User).expect("the record is in the value model"); diff --git a/languages/python/mise.toml b/languages/python/mise.toml index 70d4c393f..f0876d8fa 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -85,6 +85,9 @@ libdir="$("$PYO3_PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("L export DYLD_FALLBACK_LIBRARY_PATH="${libdir}${DYLD_FALLBACK_LIBRARY_PATH:+:$DYLD_FALLBACK_LIBRARY_PATH}" export LD_LIBRARY_PATH="${libdir}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" cargo test --locked --workspace +# The deterministic switch is for the whole process, so its test is a binary +# of its own, built with the test feature. +cargo test --locked -p stackencrypt-core --features deterministic-kms --test deterministic_switch """ [tasks."test:live"] diff --git a/languages/python/tests/_declaration_cases.py b/languages/python/tests/_declaration_cases.py index cc84ce540..97b34f782 100644 --- a/languages/python/tests/_declaration_cases.py +++ b/languages/python/tests/_declaration_cases.py @@ -480,6 +480,51 @@ class User: assert planned(User) == [("email", TextEq)] +def test_class_level_hints_are_not_fields_on_a_class_read_from_its_hints(): + # No library field list, so the fields come from the hints, as on a + # SQLAlchemy class. Each form the filter checks is here. + @stackencrypt.model("users") + class User(KeywordConstructor): + email: Annotated[str, TextEq] + registry: ClassVar[dict] = {} + bare: ClassVar = 0 + token: dataclasses.InitVar[str] = "" + + assert planned(User) == [("email", TextEq)] + + +class RefusedBase(DeclarativeBase): + pass + + +def build_a_mapped_class(kind): + if kind == "no marker": + + @stackencrypt.model("mapped_no_marker") + class MappedNoMarker(RefusedBase): + __tablename__ = "mapped_no_marker" + id: Mapped[Annotated[int, Passthrough]] = mapped_column(primary_key=True) + email: Mapped[str] + + else: + + @stackencrypt.model("mapped_two_markers") + class MappedTwoMarkers(RefusedBase): + __tablename__ = "mapped_two_markers" + id: Mapped[Annotated[int, Passthrough]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq, Passthrough]] + + +@pytest.mark.parametrize( + "kind, code", + [("no marker", "declaration_no_marker"), ("two markers", "declaration_two_markers")], +) +def test_rule_a_holds_inside_mapped(kind, code): + error = refusal(lambda: build_a_mapped_class(kind)) + assert error.code == code + assert error.field == "email" + + def test_a_subclass_is_not_declared_until_it_is_decorated(): @stackencrypt.model("users") @dataclasses.dataclass diff --git a/languages/python/tests/_sync_contract.py b/languages/python/tests/_sync_contract.py index 3fa6e9914..cf1944414 100644 --- a/languages/python/tests/_sync_contract.py +++ b/languages/python/tests/_sync_contract.py @@ -58,6 +58,32 @@ def encrypt_many(t: int) -> None: return [row for rows in sealed for row in rows] +def decrypt_from_threads(cipher: Cipher, rows: List[Dict[str, object]]) -> List[User]: + """Release ``THREADS`` threads at once, each to decrypt its own slice of + ``rows`` one call at a time. Returns every user, in row order. Raises if + any call failed.""" + per_thread = -(-len(rows) // THREADS) + opened: List[List[User]] = [[] for _ in range(THREADS)] + failures: List[str] = [] + start = threading.Barrier(THREADS) + + def decrypt_many(t: int) -> None: + start.wait() + try: + for row in rows[t * per_thread : (t + 1) * per_thread]: + opened[t].extend(cipher.decrypt([row])) + except Exception as error: + failures.append(repr(error)) + + threads = [threading.Thread(target=decrypt_many, args=(t,)) for t in range(THREADS)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert failures == [], failures + return [model for models in opened for model in models] + + def send_sigint_later(seconds: float) -> threading.Timer: """Send SIGINT to this process after ``seconds``.""" timer = threading.Timer(seconds, os.kill, (os.getpid(), signal.SIGINT)) diff --git a/languages/python/tests/test_concurrency_live.py b/languages/python/tests/test_concurrency_live.py index 3d876c612..2127fe04a 100644 --- a/languages/python/tests/test_concurrency_live.py +++ b/languages/python/tests/test_concurrency_live.py @@ -13,7 +13,14 @@ import pytest from _live import LIVE, run_python -from _sync_contract import PER_THREAD, THREADS, User, encrypt_from_threads, user +from _sync_contract import ( + PER_THREAD, + THREADS, + User, + decrypt_from_threads, + encrypt_from_threads, + user, +) from stackencrypt import Cipher @@ -34,6 +41,13 @@ def test_threads_encrypt_at_once_and_every_row_opens(): assert cipher.decrypt(rows) == [user(n) for n in range(THREADS * PER_THREAD)] +def test_threads_decrypt_at_once_and_every_row_opens(): + cipher = Cipher(User) + rows = encrypt_from_threads(cipher, PER_THREAD) + + assert decrypt_from_threads(cipher, rows) == [user(n) for n in range(THREADS * PER_THREAD)] + + @pytest.mark.skipif(sys.platform == "win32", reason="SIGINT is sent with os.kill") def test_an_interrupt_during_encrypt_waits_for_it_and_the_cipher_still_works(): result = run_python([str(HERE / "_interrupt.py")]) diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py index 8cf8442c1..573472ff4 100644 --- a/languages/python/tests/test_encrypt_live.py +++ b/languages/python/tests/test_encrypt_live.py @@ -97,6 +97,8 @@ def test_two_ciphers_share_one_engine_client(): # Read after the first call: an earlier test may have loaded the keyset # more than once, if a load failed and the next call loaded it again. loads = _engine.requests().get("load-keyset") + # A missing counter would make the last check compare None with None. + assert loads is not None and loads >= 1, _engine.requests() Cipher(Contact).encrypt([Contact(email="b@example.com")]) diff --git a/languages/python/tests/test_gate_keyboard_interrupt.py b/languages/python/tests/test_gate_keyboard_interrupt.py index 9b8c4e40f..93a78b271 100644 --- a/languages/python/tests/test_gate_keyboard_interrupt.py +++ b/languages/python/tests/test_gate_keyboard_interrupt.py @@ -30,12 +30,17 @@ def call_then_wait_for_the_interrupt(): def test_interrupt_waits_for_the_detached_call_to_return(): + # Cancelled and joined in `finally`: a timer left running would send + # SIGINT into a later, unrelated test. timer = send_sigint_later(SIGNAL_AT_S) - started = time.monotonic() - with pytest.raises(KeyboardInterrupt): - call_then_wait_for_the_interrupt() - raised_after_s = time.monotonic() - started - timer.join() + try: + started = time.monotonic() + with pytest.raises(KeyboardInterrupt): + call_then_wait_for_the_interrupt() + raised_after_s = time.monotonic() - started + finally: + timer.cancel() + timer.join() # The signal was sent at 0.2 s. If the call had been cut short, the # interrupt would arrive then. It must arrive after the call's full second. @@ -44,8 +49,11 @@ def test_interrupt_waits_for_the_detached_call_to_return(): def test_the_extension_is_usable_after_an_interrupt(): timer = send_sigint_later(SIGNAL_AT_S) - with pytest.raises(KeyboardInterrupt): - call_then_wait_for_the_interrupt() - timer.join() + try: + with pytest.raises(KeyboardInterrupt): + call_then_wait_for_the_interrupt() + finally: + timer.cancel() + timer.join() assert probe(10) >= 10 diff --git a/languages/python/tests/test_redaction.py b/languages/python/tests/test_redaction.py index 0bc5ed87d..21eb31607 100644 --- a/languages/python/tests/test_redaction.py +++ b/languages/python/tests/test_redaction.py @@ -8,7 +8,8 @@ import attrs import pydantic import pytest -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from sqlalchemy import create_engine, event +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column import stackencrypt from stackencrypt import DeclarationError, Ignore, Passthrough, TextEq @@ -80,6 +81,26 @@ def declare_sqlalchemy(): return MappedUser(email=SECRET, age=42) +def test_an_expired_or_detached_sqlalchemy_column_runs_no_select(): + # After a commit every attribute is expired: reading one runs a SELECT, + # and on a detached instance it raises. repr must do neither. + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + statements = [] + event.listen(engine, "before_cursor_execute", lambda *args: statements.append(args[2])) + expected = "MappedUser(email=, age=)" + with Session(engine) as session: + user = MappedUser(email=SECRET, age=42) + session.add(user) + session.commit() + statements.clear() + + assert repr(user) == expected + assert statements == [] + assert repr(user) == expected + assert statements == [] + + KINDS = [ declare_dataclass, declare_attrs, diff --git a/languages/python/tests_deterministic/test_concurrency.py b/languages/python/tests_deterministic/test_concurrency.py index 24161f487..9baca617f 100644 --- a/languages/python/tests_deterministic/test_concurrency.py +++ b/languages/python/tests_deterministic/test_concurrency.py @@ -7,7 +7,14 @@ live test. pyproject.toml puts `tests/` on the path. """ -from _sync_contract import PER_THREAD, THREADS, User, encrypt_from_threads, user +from _sync_contract import ( + PER_THREAD, + THREADS, + User, + decrypt_from_threads, + encrypt_from_threads, + user, +) from stackencrypt import Cipher @@ -18,3 +25,10 @@ def test_threads_encrypt_at_once_and_every_row_opens(): rows = encrypt_from_threads(cipher, PER_THREAD) assert cipher.decrypt(rows) == [user(n) for n in range(THREADS * PER_THREAD)] + + +def test_threads_decrypt_at_once_and_every_row_opens(): + cipher = Cipher(User) + rows = encrypt_from_threads(cipher, PER_THREAD) + + assert decrypt_from_threads(cipher, rows) == [user(n) for n in range(THREADS * PER_THREAD)] diff --git a/languages/python/tests_deterministic/test_switch.py b/languages/python/tests_deterministic/test_switch.py new file mode 100644 index 000000000..2189f958a --- /dev/null +++ b/languages/python/tests_deterministic/test_switch.py @@ -0,0 +1,22 @@ +"""The refusals of the switch to the deterministic client. + +conftest.py has switched this process once, before any test. A second +switch is refused, and a seed that is not 32 bytes is refused before any +switch is tried. +""" + +import pytest + +from stackencrypt import StashError, _engine + + +def test_a_second_switch_is_refused(): + with pytest.raises(StashError) as raised: + _engine._use_deterministic_keys(bytes(32)) + assert raised.value.code == "stack_test_build" + + +@pytest.mark.parametrize("length", [0, 31, 33]) +def test_a_seed_that_is_not_32_bytes_is_refused(length): + with pytest.raises(ValueError): + _engine._use_deterministic_keys(bytes(length)) From a743fdf1db571627cf19aff3048de7d66361efcc Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 15:51:33 +0900 Subject: [PATCH 05/21] fix(python): small fixes from the review, with tighter tests Each change is small. Together they close the review threads tagged as quick wins. Build and engine: - build.rs fails when git status fails. Before, it recorded a clean commit. -dirty now also covers the root Cargo.toml and the Python Cargo.lock, which choose the engine's dependency versions. - run! gives its value in both builds. Before, the test build returned from the calling function. - The allocator spy uses a poisoned lock as it is. A panic inside the allocator would abort the test binary with no message. Tests: - The value tests refuse both sides of each integer range, a float too large for float32, and a decimal mantissa one past 96 bits. The edge-value test builds the cipher outside pytest.raises. - Refusals of a hand-written or replaced __str__ and __rich_repr__, and of a changed or removed hint. - A constructor's own error reaches the caller of decrypt unchanged. A dict with no JSON form gives decrypt_malformed. - The thread helpers time out and name the failed thread. Other helpers give clearer failure messages, and the Go-row tests check lengths before zip. - The conformance test compares vector names, not counts. - The CI script tests check that their scans find something, test the triggers() helper, and hold the deterministic build's features to pyproject.toml's. - The codegen test shows that a refused override writes no file. - The ClassVar test from the last commit keeps its bare ClassVar case for Python 3.10 and later. Python 3.9 cannot resolve one written as a string. Docs and example: - AGENTS.md says which job builds the wheels, where extension-module is named, and what test:cargo runs. - G6 on the scorecard is strained: it lists two gaps with no test. - The example builds EQL before it deletes its database volume, says to trust EQL's mise file, and accepts a postgres:// URL. Addresses review threads 1405-F1, F4, 1406-F6, F7, F8, F9, 1414-F3, F5, 1415-F4, 1453-F8, F9, 1499-F17, 1509-F10, F11, 1511-F1, F3, F4, F5, 1513-F1, F4, F7, F8, 1516-F3, 1517-F1, F2, F7, F8, and 1621-F1, F2. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- languages/python/crates/core/build.rs | 27 +++++++++++-- languages/python/crates/core/src/client.rs | 19 +++++---- languages/python/crates/core/src/lib.rs | 33 ++++++++++----- .../crates/core/tests/gate_frame_reader.rs | 9 +++-- .../crates/core/tests/value_conformance.rs | 40 +++++++++++++++---- .../python/crates/engine/src/heap_hygiene.rs | 9 ++++- languages/python/docs/scorecard.md | 6 ++- .../examples/customer-directory/README.md | 6 +++ .../customer-directory/app_sqlalchemy.py | 4 +- .../examples/customer-directory/mise.toml | 5 ++- languages/python/tests/_declaration_cases.py | 5 ++- languages/python/tests/_live.py | 4 +- languages/python/tests/_readme.py | 10 ++--- languages/python/tests/_sync_contract.py | 22 +++++----- languages/python/tests/test_cipher.py | 25 ++++++++++++ languages/python/tests/test_decrypt.py | 9 +++++ languages/python/tests/test_encrypt_live.py | 25 ++++++++++++ languages/python/tests/test_readme.py | 4 +- languages/python/tests/test_redaction.py | 20 +++++----- languages/python/tests/test_values.py | 37 ++++++++++++----- .../tests_deterministic/test_go_rows.py | 3 ++ packages/eql/crates/eql-codegen/tests/cli.rs | 19 +++++++-- scripts/__tests__/eql-suite-ci.test.mjs | 8 ++++ scripts/__tests__/python-ci.test.mjs | 31 ++++++++++++++ scripts/__tests__/workflows-lib.test.mjs | 25 ++++++++++++ 26 files changed, 326 insertions(+), 81 deletions(-) create mode 100644 scripts/__tests__/workflows-lib.test.mjs diff --git a/AGENTS.md b/AGENTS.md index 5e52f8ed8..8aa75cf54 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin, never in a crate manifest, so `cargo test` links libpython. `pyproject.toml` sets it, and `test:deterministic` names it again in its `--features` list; `scripts/__tests__/python-ci.test.mjs` holds the two lists together. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It also runs one test binary of `crates/core`, `deterministic_switch`, with the `deterministic-kms` feature: the switch is for the whole process, so that test needs a process of its own. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. A third job, `wheels`, builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `languages/python/examples/customer-directory`: An internal FastAPI example of the Python SDK, written twice: with psycopg (`app.py`) and with SQLAlchemy (`app_sqlalchemy.py`). It has its own `pyproject.toml` and `uv.lock`, and installs `stackencrypt` from this checkout. Its tests need real credentials, and CI does not run them. diff --git a/languages/python/crates/core/build.rs b/languages/python/crates/core/build.rs index 80d873f9a..5aa0aa46c 100644 --- a/languages/python/crates/core/build.rs +++ b/languages/python/crates/core/build.rs @@ -8,9 +8,13 @@ //! was not built from. The scope is every stack-* crate, not only //! `stack-encrypt`: it depends by path on `stack-kms` and //! `stack-encrypt-derive`, and a wider scope can only add a `-dirty` that is -//! true. Edits outside `packages/stack-*` (this workspace, the Python -//! package) are not engine edits and do not mark the engine dirty; the wheel -//! version and the checkout commit describe those. +//! true. Two more files choose the engine's code: the root `Cargo.toml`, +//! which sets the versions of the crates the engine takes with +//! `workspace = true`, and this workspace's `Cargo.lock`, which holds the +//! exact versions. An edit to either also marks the engine dirty. Other +//! edits outside `packages/stack-*` (this workspace, the Python package) are +//! not engine edits; the wheel version and the checkout commit describe +//! those. //! //! `STACKENCRYPT_ENGINE_COMMIT` in the environment wins over git, for a build //! from a source tree that is not a checkout. It is trusted as given: with no @@ -26,6 +30,9 @@ const ENV_OVERRIDE: &str = "STACKENCRYPT_ENGINE_COMMIT"; /// `stack-*` entry in it is engine source for the `-dirty` check. const PACKAGES_DIR: &str = "../../../../packages"; const ENGINE_PREFIX: &str = "stack-"; +/// The files outside the stack-* crates that choose the engine's dependency +/// versions, relative to this crate. +const DEPENDENCY_FILES: [&str; 2] = ["../../../../Cargo.toml", "../../Cargo.lock"]; fn main() { println!("cargo:rerun-if-env-changed={ENV_OVERRIDE}"); @@ -81,6 +88,12 @@ fn commit_from_git() -> String { for dir in engine_dirs(&manifest_dir) { println!("cargo:rerun-if-changed={}", dir.display()); } + for file in DEPENDENCY_FILES { + println!( + "cargo:rerun-if-changed={}", + manifest_dir.join(file).display() + ); + } let commit = git(&["rev-parse", "HEAD"]).unwrap_or_else(|| { panic!( @@ -101,9 +114,15 @@ fn commit_from_git() -> String { dirs.iter() .map(|d| format!(":(exclude){}/fuzz", d.display())), ); + paths.extend(DEPENDENCY_FILES.iter().map(|file| (*file).to_owned())); let mut status = vec!["status", "--porcelain", "--"]; status.extend(paths.iter().map(String::as_str)); - let dirty = git(&status).is_some_and(|s| !s.is_empty()); + // A status that fails cannot say the tree is clean, so it fails the + // build, as `rev-parse` does. + let porcelain = git(&status).unwrap_or_else(|| { + panic!("`git status` failed, so the build cannot tell whether the engine has edits") + }); + let dirty = !porcelain.is_empty(); if dirty { format!("{commit}-dirty") } else { diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index 3d87b20f4..a86d59dca 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -92,16 +92,21 @@ pub fn shared() -> Result<&'static Client, Refusal> { /// after the switch. The two have different key source types, so a function /// that returned "the current client" would still leave each operation to /// write a branch per client. +/// +/// Both branches give the macro's value: the test branch leaves the block, +/// not the function, so code after `run!` runs in the test build too. macro_rules! run { (|$cipher:ident| $future:expr) => {{ - #[cfg(feature = "deterministic-kms")] - if let Some((cipher, runtime)) = $crate::client::deterministic() { - let $cipher = cipher; - return runtime.block_on($future); + 'run: { + #[cfg(feature = "deterministic-kms")] + if let Some((cipher, runtime)) = $crate::client::deterministic() { + let $cipher = cipher; + break 'run runtime.block_on($future); + } + let client = $crate::client::shared()?; + let $cipher = client.cipher(); + client.block_on($future) } - let client = $crate::client::shared()?; - let $cipher = client.cipher(); - client.block_on($future) }}; } pub(crate) use run; diff --git a/languages/python/crates/core/src/lib.rs b/languages/python/crates/core/src/lib.rs index 98a165c56..c8273e59f 100644 --- a/languages/python/crates/core/src/lib.rs +++ b/languages/python/crates/core/src/lib.rs @@ -84,9 +84,10 @@ mod tests { } /// The commit is this checkout's, with `-dirty` exactly when a file - /// under a `packages/stack-*` crate has changed outside its `fuzz/`. - /// The status is read for all of `packages/` and filtered here, so the - /// test does not repeat build.rs's pathspec. + /// under a `packages/stack-*` crate has changed outside its `fuzz/`, or + /// the root `Cargo.toml` or this workspace's `Cargo.lock` has changed. + /// The status is read for wider paths and filtered here, so the test + /// does not repeat build.rs's pathspec. #[test] fn engine_commit_is_this_checkouts_commit() { // A commit given in STACKENCRYPT_ENGINE_COMMIT is trusted as given. @@ -96,19 +97,29 @@ mod tests { let Some(head) = git(&["rev-parse", "HEAD"]) else { return; }; - let status = git(&["status", "--porcelain", "--", "../../../../packages"]) - .expect("git status runs in a checkout"); + let status = git(&[ + "status", + "--porcelain", + "--", + "../../../../packages", + "../../../../Cargo.toml", + "../../Cargo.lock", + ]) + .expect("git status runs in a checkout"); // A porcelain line is a two-letter status, a space and the path from // the repository root. A rename shows "old -> new". let dirty = status.lines().any(|line| { line.get(3..).is_some_and(|paths| { paths.split(" -> ").any(|path| { - path.trim_matches('"') - .strip_prefix("packages/") - .and_then(|rest| rest.split_once('/')) - .is_some_and(|(krate, rest)| { - krate.starts_with("stack-") && !rest.starts_with("fuzz/") - }) + let path = path.trim_matches('"'); + path == "Cargo.toml" + || path == "languages/python/Cargo.lock" + || path + .strip_prefix("packages/") + .and_then(|rest| rest.split_once('/')) + .is_some_and(|(krate, rest)| { + krate.starts_with("stack-") && !rest.starts_with("fuzz/") + }) }) }) }); diff --git a/languages/python/crates/core/tests/gate_frame_reader.rs b/languages/python/crates/core/tests/gate_frame_reader.rs index d444b47a1..13ae7196b 100644 --- a/languages/python/crates/core/tests/gate_frame_reader.rs +++ b/languages/python/crates/core/tests/gate_frame_reader.rs @@ -2,9 +2,12 @@ //! frame and recover each field's value kind, with only the public API of //! `stack-encrypt`'s `dynamic` module. //! -//! The record is built with `ValueFrame::of`, the same type and writer -//! `dynamic::decrypt` returns its opened records in. This test does not -//! decrypt: it proves the reading half, which is the half the extension owns. +//! The record is built with `ValueFrame::of`, the same type +//! `dynamic::decrypt` returns its opened records in. The writer is not the +//! same: `ValueFrame::of` writes through the value model's `Modeled` writer, +//! and `decrypt` writes each record with a plain `FrameWriter`. So this test +//! does not check the bytes of a real decrypt result. It does not decrypt: +//! it proves the reading half, which is the half the extension owns. use stack_encrypt::dynamic::frame::Frame; use stack_encrypt::dynamic::plaintext::{ diff --git a/languages/python/crates/core/tests/value_conformance.rs b/languages/python/crates/core/tests/value_conformance.rs index f06e9221c..4cf16513b 100644 --- a/languages/python/crates/core/tests/value_conformance.rs +++ b/languages/python/crates/core/tests/value_conformance.rs @@ -21,6 +21,31 @@ const CORPUS: &str = include_str!("../testdata/value-conformance.json"); /// (`int8`, the unsigned and 128-bit integers, `uuid`), it is not a scalar /// (`array`, `object`), or it is a timestamp in a unit other than the /// nanoseconds the SDK writes. +/// The vectors the SDK writes, in corpus order: every scalar kind it has, +/// from the corpus at this commit. A vector added on a refresh fails the +/// test with its name. +const WRITTEN: &[&str] = &[ + "false", + "true", + "int16", + "int32", + "int64", + "float32", + "float64", + "text", + "text-nul", + "bytes", + "date", + "date-year-zero", + "timestamp", + "timestamp-nanos", + "timestamp-expanded-year", + "timestamp-negative-year", + "decimal", + "decimal-min", + "decimal-small", +]; + const NOT_WRITTEN: &[&str] = &[ "null", "undefined", @@ -155,7 +180,7 @@ fn decimal(text: &str) -> (i128, u8) { #[test] fn every_vector_the_sdk_writes_gives_its_bytes_and_reads_back() { let corpus = corpus(); - let mut written = 0; + let mut written = Vec::new(); for vector in corpus["vectors"].as_array().expect("vectors") { let name = vector["name"].as_str().expect("a name"); let (mut text, mut bytes) = (String::new(), Vec::new()); @@ -185,10 +210,9 @@ fn every_vector_the_sdk_writes_gives_its_bytes_and_reads_back() { expected, "{name}: the read value" ); - written += 1; + written.push(name); } - // Every scalar kind the SDK has, from the corpus at this commit. - assert_eq!(written, 19); + assert_eq!(written, WRITTEN); } #[test] @@ -204,7 +228,7 @@ fn every_malformed_vector_is_refused() { #[test] fn every_normalized_vector_reads_as_its_canonical_value() { let corpus = corpus(); - let mut written = 0; + let mut written = Vec::new(); for vector in corpus["normalized"].as_array().expect("normalized") { let name = vector["name"].as_str().expect("a name"); let transport = unhex(vector["transport"].as_str().expect("transport")); @@ -245,10 +269,10 @@ fn every_normalized_vector_reads_as_its_canonical_value() { &reencode[..], "{name}: the written bytes" ); - written += 1; + written.push(name); } } } - // decimal-negative-zero, at this commit. - assert_eq!(written, 1); + // The normalized vectors that are not a typed null, at this commit. + assert_eq!(written, ["decimal-negative-zero"]); } diff --git a/languages/python/crates/engine/src/heap_hygiene.rs b/languages/python/crates/engine/src/heap_hygiene.rs index 1d84b7285..759ac1a63 100644 --- a/languages/python/crates/engine/src/heap_hygiene.rs +++ b/languages/python/crates/engine/src/heap_hygiene.rs @@ -133,11 +133,16 @@ unsafe impl GlobalAlloc for Spy { let trace = std::backtrace::Backtrace::force_capture().to_string(); // Linux writes `<…::CipherKey>::seal`, macOS `…::CipherKey::seal`. if trace.replace(['<', '>'], "").contains(UPSTREAM_SEAL) { - *UPSTREAM.lock().unwrap() += 1; + // A panic here, inside the allocator, would abort the whole + // test binary with no message, so a poisoned lock is used + // as it is, as GATE is. + *UPSTREAM + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) += 1; } else { FOUND .lock() - .unwrap() + .unwrap_or_else(|poisoned| poisoned.into_inner()) .push(format!("{} bytes freed unwiped:\n{trace}", layout.size())); } BUSY.with(|busy| busy.set(false)); diff --git a/languages/python/docs/scorecard.md b/languages/python/docs/scorecard.md index fdd5a4bab..ed05fb74f 100644 --- a/languages/python/docs/scorecard.md +++ b/languages/python/docs/scorecard.md @@ -17,7 +17,7 @@ Each principle is marked: | G3. Run each check at the earliest stage the language allows | strained | | G4. One declaration serves the write, the query and the read | held | | G5. Batch by default | held | -| G6. A second way in has to earn its place | held | +| G6. A second way in has to earn its place | strained | | G7. State the design, and run its claims | held | | G8. Usage first, then reference | held | @@ -59,7 +59,9 @@ Each principle is marked: - Each query method is the single-value call the principle allows: one field, one search value. It makes no key request after the first call of the process. - A model's fields of different types go in one request. -## G6. A second way in has to earn its place: held +## G6. A second way in has to earn its place: strained + +Strained, not held: the two gaps below have no test. - There is one way to declare a model: `stackencrypt.model`. Every class library goes through it. `tests/_declaration_cases.py` declares one model with each of dataclasses, attrs, Pydantic, `NamedTuple`, and SQLAlchemy. - No test shows that each refusal fires for every one of the five libraries. diff --git a/languages/python/examples/customer-directory/README.md b/languages/python/examples/customer-directory/README.md index b769ab58c..218cb9b01 100644 --- a/languages/python/examples/customer-directory/README.md +++ b/languages/python/examples/customer-directory/README.md @@ -15,6 +15,12 @@ The SDK is not released, so its API can change. This example installs the SDK fr You need Docker, mise, and a Rust toolchain. mise installs the toolchain. +`mise run setup` builds EQL with EQL's own mise file, which installs its own Rust toolchain. Trust that file once, from this folder: + +```sh +mise trust ../../../../packages/eql +``` + You also need CipherStash credentials. Set `CS_CLIENT_ACCESS_KEY`, `CS_WORKSPACE_CRN`, `CS_CLIENT_ID`, and `CS_CLIENT_KEY`. Or log in with `stash auth login`. ## Run it diff --git a/languages/python/examples/customer-directory/app_sqlalchemy.py b/languages/python/examples/customer-directory/app_sqlalchemy.py index e0011bfe5..314329ff3 100644 --- a/languages/python/examples/customer-directory/app_sqlalchemy.py +++ b/languages/python/examples/customer-directory/app_sqlalchemy.py @@ -26,8 +26,10 @@ # hide_parameters leaves the bound values out of SQLAlchemy's error text. # PostgreSQL's own detail can still quote the start of a rejected value. +# make_url reads both postgresql:// and postgres://, as app.py accepts both. +# The URL object, not its str(), which writes the password as ***. engine = sa.create_engine( - os.environ["DATABASE_URL"].replace("postgresql://", "postgresql+psycopg://", 1), + sa.engine.make_url(os.environ["DATABASE_URL"]).set(drivername="postgresql+psycopg"), hide_parameters=True, ) diff --git a/languages/python/examples/customer-directory/mise.toml b/languages/python/examples/customer-directory/mise.toml index 609c5eee3..e9838414c 100644 --- a/languages/python/examples/customer-directory/mise.toml +++ b/languages/python/examples/customer-directory/mise.toml @@ -9,10 +9,11 @@ description = "Reset the example's database, install EQL v4 and the table, and i shell = "bash -c" run = """ set -euo pipefail -docker compose down --volumes -docker compose up --detach --wait # Built from the tree, because the released copy moves only at a version bump. +# Built first, so a failed build leaves the database as it was. mise -C ../../../../packages/eql run build +docker compose down --volumes +docker compose up --detach --wait psql_in_db() { docker compose exec -T db psql -U postgres -d customer_directory -v ON_ERROR_STOP=1 -q } diff --git a/languages/python/tests/_declaration_cases.py b/languages/python/tests/_declaration_cases.py index 97b34f782..82a533354 100644 --- a/languages/python/tests/_declaration_cases.py +++ b/languages/python/tests/_declaration_cases.py @@ -487,8 +487,11 @@ def test_class_level_hints_are_not_fields_on_a_class_read_from_its_hints(): class User(KeywordConstructor): email: Annotated[str, TextEq] registry: ClassVar[dict] = {} - bare: ClassVar = 0 token: dataclasses.InitVar[str] = "" + if sys.version_info >= (3, 10): + # Python 3.9 cannot resolve a bare ClassVar written as a string, + # as the future import writes it: get_type_hints raises. + bare: ClassVar = 0 assert planned(User) == [("email", TextEq)] diff --git a/languages/python/tests/_live.py b/languages/python/tests/_live.py index cd1726ab1..a58dcac93 100644 --- a/languages/python/tests/_live.py +++ b/languages/python/tests/_live.py @@ -62,4 +62,6 @@ def run_python(args: List[str], env: Optional[Dict[str, str]] = None) -> Any: timeout=120, ) assert done.returncode == 0, done.stderr - return json.loads(done.stdout.strip().splitlines()[-1]) + lines = done.stdout.strip().splitlines() + assert lines, f"the child printed nothing. Its stderr:\n{done.stderr}" + return json.loads(lines[-1]) diff --git a/languages/python/tests/_readme.py b/languages/python/tests/_readme.py index 1764a428c..a3499133b 100644 --- a/languages/python/tests/_readme.py +++ b/languages/python/tests/_readme.py @@ -53,16 +53,16 @@ def program() -> str: class _TopLevel(ast.NodeVisitor): - """The names a program binds at its top level, and the names it reads - there. A function, class, lambda, or comprehension has a scope of its - own, so its names are skipped.""" + """The names a program binds at its top level. A function, class, + lambda, or comprehension has a scope of its own, so its names are + skipped.""" def __init__(self) -> None: self.bound: Set[str] = set() - self.read: Set[str] = set() def visit_Name(self, node: ast.Name) -> None: - (self.bound if isinstance(node.ctx, ast.Store) else self.read).add(node.id) + if isinstance(node.ctx, ast.Store): + self.bound.add(node.id) def visit_alias(self, node: ast.alias) -> None: self.bound.add((node.asname or node.name).split(".")[0]) diff --git a/languages/python/tests/_sync_contract.py b/languages/python/tests/_sync_contract.py index cf1944414..6d3564d75 100644 --- a/languages/python/tests/_sync_contract.py +++ b/languages/python/tests/_sync_contract.py @@ -13,6 +13,7 @@ import signal import threading import time +import traceback from typing import Annotated, Dict, List import stackencrypt @@ -39,15 +40,16 @@ def encrypt_from_threads(cipher: Cipher, per_thread: int) -> List[Dict[str, obje any call failed.""" sealed: List[List[Dict[str, object]]] = [[] for _ in range(THREADS)] failures: List[str] = [] - start = threading.Barrier(THREADS) + # A timeout, so one thread that never arrives fails the test, not hangs it. + start = threading.Barrier(THREADS, timeout=30) def encrypt_many(t: int) -> None: - start.wait() try: + start.wait() for n in range(per_thread): sealed[t].extend(cipher.encrypt([user(t * per_thread + n)])) - except Exception as error: - failures.append(repr(error)) + except Exception: + failures.append(f"thread {t}: {traceback.format_exc()}") threads = [threading.Thread(target=encrypt_many, args=(t,)) for t in range(THREADS)] for thread in threads: @@ -65,15 +67,15 @@ def decrypt_from_threads(cipher: Cipher, rows: List[Dict[str, object]]) -> List[ per_thread = -(-len(rows) // THREADS) opened: List[List[User]] = [[] for _ in range(THREADS)] failures: List[str] = [] - start = threading.Barrier(THREADS) + start = threading.Barrier(THREADS, timeout=30) def decrypt_many(t: int) -> None: - start.wait() try: + start.wait() for row in rows[t * per_thread : (t + 1) * per_thread]: opened[t].extend(cipher.decrypt([row])) - except Exception as error: - failures.append(repr(error)) + except Exception: + failures.append(f"thread {t}: {traceback.format_exc()}") threads = [threading.Thread(target=decrypt_many, args=(t,)) for t in range(THREADS)] for thread in threads: @@ -96,4 +98,6 @@ def wait_for_the_interrupt() -> None: loop gives it that check, and ends if it never comes.""" deadline = time.monotonic() + 5 while time.monotonic() < deadline: - pass + # A short sleep still returns to Python, which then checks for the + # signal, and it does not keep a core busy. + time.sleep(0.01) diff --git a/languages/python/tests/test_cipher.py b/languages/python/tests/test_cipher.py index 7876481a5..fdd7f40be 100644 --- a/languages/python/tests/test_cipher.py +++ b/languages/python/tests/test_cipher.py @@ -128,6 +128,31 @@ def test_a_replaced_repr_is_refused(credentials): assert "__repr__ replaced" in str(error) +@pytest.mark.parametrize("change", ["changed", "removed"]) +def test_a_changed_or_removed_hint_is_refused(credentials, change): + User = declared() + if change == "changed": + User.__annotations__["age"] = Annotated[str, Passthrough] + else: + del User.__annotations__["age"] + + error = refusal(lambda: Cipher(User)) + assert error.code == "declaration_class_changed" + assert error.reason == "fields" + assert error.field == "age" + + +@pytest.mark.parametrize("name", ["__str__", "__rich_repr__"]) +def test_each_replaced_method_is_refused(credentials, name): + User = declared() + setattr(User, name, lambda self: self.email) + + error = refusal(lambda: Cipher(User)) + assert error.code == "declaration_class_changed" + assert error.reason == "methods" + assert f"{name} replaced" in str(error) + + def test_a_hint_that_no_longer_resolves_names_the_name(credentials): User = declared() User.__annotations__["posts"] = "List[Undefined]" diff --git a/languages/python/tests/test_decrypt.py b/languages/python/tests/test_decrypt.py index 6a01f03f3..01e1dd190 100644 --- a/languages/python/tests/test_decrypt.py +++ b/languages/python/tests/test_decrypt.py @@ -65,6 +65,15 @@ def test_one_row_on_its_own_is_refused_as_one_row(untouched): assert "not one row" in error.message +def test_a_dict_that_is_not_json_is_malformed(untouched): + # A set has no JSON form: json.dumps raises TypeError. + value = {"i": {"t": "users", "c": "email"}, "v": {1, 2}} + error = refused([{"email": value, "age": 3}]) + assert error.code == "decrypt_malformed" + assert error.field == "email" + assert "row 0" in error.message + + def test_a_str_that_is_not_valid_utf8_is_malformed(untouched): # A lone surrogate: a str Python holds, with no UTF-8 encoding. error = refused([{"email": "\ud800", "age": 3}]) diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py index 573472ff4..82bf0c06f 100644 --- a/languages/python/tests/test_encrypt_live.py +++ b/languages/python/tests/test_encrypt_live.py @@ -52,6 +52,31 @@ class Contact: email: Annotated[str, TextEq] +class Refused(Exception): + """The model's own error, from its constructor.""" + + +@stackencrypt.model("contacts") +@dataclasses.dataclass +class StrictContact: + """Contact's table and column, with a constructor that refuses a value.""" + + email: Annotated[str, TextEq] + + def __post_init__(self): + if self.email == "bad@example.com": + raise Refused("the model refused this email") + + +def test_an_error_from_the_constructor_reaches_the_caller_unchanged(): + rows = Cipher(Contact).encrypt( + [Contact(email="good@example.com"), Contact(email="bad@example.com")] + ) + + with pytest.raises(Refused, match="the model refused this email"): + Cipher(StrictContact).decrypt(rows) + + @needs_database def test_an_encrypted_row_inserts_with_a_plain_parameter(): psycopg = pytest.importorskip("psycopg") diff --git a/languages/python/tests/test_readme.py b/languages/python/tests/test_readme.py index 008f27456..fc57f6986 100644 --- a/languages/python/tests/test_readme.py +++ b/languages/python/tests/test_readme.py @@ -33,6 +33,8 @@ def test_the_readme_has_sql_for_the_live_test_to_run(): def test_the_readme_installs_the_bundle_the_live_tests_install(): # The live task cannot run the README's psql command from the root, so it # runs its own. Both must name the same bundle. - [install] = [block for block in blocks("sh") if "psql" in block] + installs = [block for block in blocks("sh") if "psql" in block] + assert len(installs) == 1, "the README must have exactly one psql install block" + [install] = installs assert BUNDLE in install assert f"../../{BUNDLE}" in MISE.read_text(encoding="utf-8") diff --git a/languages/python/tests/test_redaction.py b/languages/python/tests/test_redaction.py index 21eb31607..f4db8be87 100644 --- a/languages/python/tests/test_redaction.py +++ b/languages/python/tests/test_redaction.py @@ -275,17 +275,17 @@ def test_a_hand_written_method_in_any_form_is_refused(form): assert error.code == "declaration_repr_present" -def test_a_hand_written_str_is_refused(): - def build(): - @stackencrypt.model("users") - @dataclasses.dataclass - class User: - email: Annotated[str, TextEq] - - def __str__(self): - return self.email +@pytest.mark.parametrize("name", ["__str__", "__rich_repr__"]) +def test_each_hand_written_method_is_refused(name): + def method(self): + return self.email - assert pytest.raises(DeclarationError, build).value.code == "declaration_repr_present" + cls = dataclasses.make_dataclass( + "User", [("email", Annotated[str, TextEq])], namespace={name: method} + ) + error = pytest.raises(DeclarationError, stackencrypt.model("users"), cls).value + assert error.code == "declaration_repr_present" + assert name in str(error) def test_redact_false_keeps_a_hand_written_repr_and_shows_plaintext(): diff --git a/languages/python/tests/test_values.py b/languages/python/tests/test_values.py index a9a4ed9ec..0e7821a23 100644 --- a/languages/python/tests/test_values.py +++ b/languages/python/tests/test_values.py @@ -24,10 +24,12 @@ Boolean, Cipher, DateEq, + DeclarationError, DoubleEq, IntegerEq, InvalidValueError, NumericEq, + QueryError, RealEq, SmallintEq, StashError, @@ -55,18 +57,21 @@ } -def through_encrypt(marker: Any, value: Any) -> None: +def declared(marker: Any) -> Tuple[Cipher, Any]: + """A cipher for a class with one field ``x`` of ``marker``. Built + outside ``pytest.raises``, so a refused declaration fails the test.""" cls = dataclasses.make_dataclass( "Row", [("x", Annotated[PYTHON_TYPE[marker], marker])] ) - Cipher(stackencrypt.model("rows")(cls)).encrypt([cls(x=value)]) + return Cipher(stackencrypt.model("rows")(cls)), cls -def through_eq(marker: Any, value: Any) -> None: - cls = dataclasses.make_dataclass( - "Row", [("x", Annotated[PYTHON_TYPE[marker], marker])] - ) - Cipher(stackencrypt.model("rows")(cls)).eq("x", value) +def through_encrypt(cipher: Cipher, cls: Any, value: Any) -> None: + cipher.encrypt([cls(x=value)]) + + +def through_eq(cipher: Cipher, cls: Any, value: Any) -> None: + cipher.eq("x", value) def calls(cases: List[Tuple[Any, ...]]) -> List[Any]: @@ -85,9 +90,15 @@ def calls(cases: List[Tuple[Any, ...]]) -> List[Any]: (BigintEq, 2**63, "value_out_of_range", "int64"), (BigintEq, -(2**63) - 1, "value_out_of_range", "int64"), (IntegerEq, 2**31, "value_out_of_range", "int32"), + (IntegerEq, -(2**31) - 1, "value_out_of_range", "int32"), + (SmallintEq, 2**15, "value_out_of_range", "int16"), (SmallintEq, -(2**15) - 1, "value_out_of_range", "int16"), (RealEq, 0.1, "value_not_representable", "float32"), + (RealEq, 1e39, "value_not_representable", "float32"), (NumericEq, decimal.Decimal("NaN"), "value_not_representable", "decimal"), + # One past the largest 96-bit mantissa, on each side. + (NumericEq, decimal.Decimal("79228162514264337593543950336"), "value_not_representable", "decimal"), + (NumericEq, decimal.Decimal("-79228162514264337593543950336"), "value_not_representable", "decimal"), (NumericEq, decimal.Decimal("Infinity"), "value_not_representable", "decimal"), (NumericEq, decimal.Decimal("1e-29"), "value_not_representable", "decimal"), (TimestampEq, datetime.datetime(2024, 1, 1), "value_naive_datetime", "timestamp"), @@ -107,8 +118,9 @@ def calls(cases: List[Tuple[Any, ...]]) -> List[Any]: def test_a_value_that_does_not_fit_is_refused_before_any_request( untouched, call, marker, value, code, reason ): + cipher, cls = declared(marker) with pytest.raises(InvalidValueError) as raised: - call(marker, value) + call(cipher, cls, value) assert raised.value.code == code assert raised.value.field == "x" @@ -120,6 +132,8 @@ def test_a_value_that_does_not_fit_is_refused_before_any_request( (BigintEq, 2**63 - 1), (BigintEq, -(2**63)), (IntegerEq, 2**31 - 1), + (IntegerEq, -(2**31)), + (SmallintEq, 2**15 - 1), (SmallintEq, -(2**15)), (RealEq, 0.5), (RealEq, math.nan), @@ -141,7 +155,10 @@ def test_a_value_that_does_not_fit_is_refused_before_any_request( @pytest.mark.parametrize(("call", "marker", "value"), calls(ACCEPTED)) def test_a_value_at_the_edge_passes_every_check(call, marker, value): + cipher, cls = declared(marker) with pytest.raises(StashError) as raised: - call(marker, value) + call(cipher, cls, value) - assert not isinstance(raised.value, InvalidValueError), raised.value + # Only the failure to build the client shows that every check passed. + refused = (InvalidValueError, QueryError, DeclarationError) + assert not isinstance(raised.value, refused), raised.value diff --git a/languages/python/tests_deterministic/test_go_rows.py b/languages/python/tests_deterministic/test_go_rows.py index 429ff4463..d458ef27b 100644 --- a/languages/python/tests_deterministic/test_go_rows.py +++ b/languages/python/tests_deterministic/test_go_rows.py @@ -201,6 +201,8 @@ def test_python_opens_the_rows_the_go_sdk_stored(fixture, cls, ids): opened = Cipher(cls).decrypt([stored_row(cls, row) for row in rows]) + # zip stops at the shorter list, and Python 3.9 has no strict=True. + assert len(opened) == len(rows) for row, model in zip(rows, opened): expected = model_of(cls, row) for field in dataclasses.fields(cls): @@ -214,6 +216,7 @@ def test_python_seals_what_the_go_sdk_sealed_apart_from_each_ciphertext(fixture, sealed = Cipher(cls).encrypt([model_of(cls, row) for row in rows]) + assert len(sealed) == len(rows) for row, columns in zip(rows, sealed): for field in sealed_fields(cls): assert without_ciphertext(columns[field.name]) == without_ciphertext( diff --git a/packages/eql/crates/eql-codegen/tests/cli.rs b/packages/eql/crates/eql-codegen/tests/cli.rs index 494cc8672..a2f2ecd9b 100644 --- a/packages/eql/crates/eql-codegen/tests/cli.rs +++ b/packages/eql/crates/eql-codegen/tests/cli.rs @@ -223,14 +223,18 @@ fn select_subcommand_drops_files_confined_to_the_other_version() { /// `go-eql` and `py-eql` write outside the EQL subtree, so an /// `EQL_CODEGEN_OUT_ROOT` override would send them outside the throwaway tree. -/// Each refuses the override with exit 2, before writing anything. +/// Each refuses the override with exit 2, before writing anything. The +/// override is two folders deep, so the emitters' `../../` paths would land +/// inside the throwaway tree, where the test looks for them. #[test] fn cross_tree_emitters_refuse_the_output_root_override() { - let out_root = tempdir(); + let tree = tempdir(); + let out_root = tree.0.join("one").join("two"); + std::fs::create_dir_all(&out_root).expect("create the nested output root"); for command in ["go-eql", "py-eql"] { let out = Command::new(bin()) .arg(command) - .env("EQL_CODEGEN_OUT_ROOT", out_root.0.as_os_str()) + .env("EQL_CODEGEN_OUT_ROOT", out_root.as_os_str()) .output() .unwrap_or_else(|e| panic!("run eql-codegen {command}: {e}")); let stderr = String::from_utf8_lossy(&out.stderr); @@ -240,6 +244,15 @@ fn cross_tree_emitters_refuse_the_output_root_override() { stderr.contains("EQL_CODEGEN_OUT_ROOT"), "{command}: {stderr}" ); + for path in [ + eql_codegen::go_eql::GO_EQL_PATH, + eql_codegen::py_eql::PY_EQL_PATH, + ] { + assert!( + !out_root.join(path).exists(), + "{command} wrote {path} before it refused" + ); + } } } diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index aa73d3bae..8941c20e9 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -233,6 +233,14 @@ describe('a `mise run` counts as an EQL task only from inside packages/eql', () expect(eqlRunBodies(wf)).toEqual([]) }) + it('does not read a sibling folder with the same prefix as packages/eql', () => { + const wf = workflow({ + sibling: { steps: [step('mise run test:sqlx', 'packages/eql-bench')] }, + slash: { steps: [step('mise run test:schema', 'packages/eql/')] }, + }) + expect(eqlRunBodies(wf)).toEqual(['mise run test:schema']) + }) + it('strips shell comment lines, so a mention in a comment is not a run', () => { const wf = workflow({ eql: { diff --git a/scripts/__tests__/python-ci.test.mjs b/scripts/__tests__/python-ci.test.mjs index b020e43f5..8e038038d 100644 --- a/scripts/__tests__/python-ci.test.mjs +++ b/scripts/__tests__/python-ci.test.mjs @@ -117,6 +117,14 @@ describe('tests-python.yml', () => { ).toEqual([]) }) + it('finds the stack crates as path dependencies, so the filter check is not empty', () => { + // A lockfile parser that finds nothing would leave the check above with + // no tree to look for, and it would pass. + expect(pathDependencyTrees()).toEqual( + expect.arrayContaining(['packages/stack-encrypt', 'packages/stack-kms']), + ) + }) + it('runs on pull requests touching a file the core crate compiles by #[path]', () => { const paths = triggers(readWorkflow(WORKFLOW)).pull_request?.paths ?? [] const files = pathModuleFiles() @@ -208,6 +216,12 @@ describe('tests-python.yml', () => { const compiles = (step) => /\bcargo\b|\bmise run\b/.test(step.run ?? '') || /^PyO3\/maturin-action@/.test(step.uses ?? '') + // The scan must find the jobs that compile, or the check below passes + // over none: a wheel step in another form would drop out of it. + const compiling = Object.entries(wf.jobs) + .filter(([, job]) => job.steps.some(compiles)) + .map(([id]) => id) + expect(compiling.sort()).toEqual(['live', 'python', 'wheels']) const missing = Object.entries(wf.jobs) .filter(([, job]) => job.steps.some(compiles)) .filter( @@ -223,6 +237,23 @@ describe('tests-python.yml', () => { ).toEqual([]) }) + it('builds the deterministic test extension with the features of the normal build', () => { + // `--features` replaces pyproject.toml's list. A feature added there and + // not here would leave tests_deterministic/ testing another extension. + const pyproject = read(`${PYTHON_ROOT}/pyproject.toml`) + const declared = pyproject.match(/^features = \[([^\]]*)\]/m) + expect( + declared, + 'pyproject.toml has a [tool.maturin] features list', + ).not.toBeNull() + const normal = [...declared[1].matchAll(/"([^"]+)"/g)].map((m) => m[1]) + const task = read(`${PYTHON_ROOT}/mise.toml`).match(/--features ([^\s"]+)/) + expect(task, 'test:deterministic passes --features').not.toBeNull() + expect(task[1].split(',').sort()).toEqual( + [...normal, 'deterministic-kms'].sort(), + ) + }) + it('runs every task of the Python workspace, and the two cargo lints, from that folder', () => { // "A check nothing invokes reads exactly like a check that passes" // (crates-ci.test.mjs). That file scans the root mise.toml and the crate diff --git a/scripts/__tests__/workflows-lib.test.mjs b/scripts/__tests__/workflows-lib.test.mjs new file mode 100644 index 000000000..691209edf --- /dev/null +++ b/scripts/__tests__/workflows-lib.test.mjs @@ -0,0 +1,25 @@ +import yaml from 'js-yaml' +import { describe, expect, it } from 'vitest' +import { triggers } from './lib/workflows.mjs' + +// Seven workflow guards read their triggers through this helper. A change +// that drops a trigger would make each of them check fewer workflows, and +// pass. +describe('triggers()', () => { + it('reads `on:` under the YAML 1.1 `true` key and under a quoted key', () => { + expect(triggers(yaml.load('on:\n push: {}'))).toHaveProperty('push') + expect(triggers(yaml.load("'on':\n push: {}"))).toHaveProperty('push') + }) + + it('keeps a trigger declared with no value', () => { + expect( + 'workflow_dispatch' in triggers(yaml.load('on:\n workflow_dispatch:')), + ).toBe(true) + }) + + it('gives {} for a missing or non-object `on:`', () => { + expect(triggers(yaml.load('name: x'))).toEqual({}) + expect(triggers(yaml.load('on: push'))).toEqual({}) + expect(triggers(undefined)).toEqual({}) + }) +}) From 8593093ee24cba922544abaddf11a0fcd88ed3f9 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 16:18:40 +0900 Subject: [PATCH 06/21] fix(python): say how to fix a bare ClassVar on Python 3.9 Under from __future__ import annotations, Python 3.9 cannot read a bare ClassVar hint: get_type_hints raises TypeError. The decorator reported it as an unresolved name, and told the user to define or import it. Nothing was missing, so that advice did not help. The decorator now evaluates the hint that failed. If it is ClassVar with no type, the error says to write ClassVar[int], with the reason bare_class_var. The evaluation is Python's own, so typing.ClassVar and an alias are found too. Python versions that read the bare form accept it as before. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .../python/stackencrypt/_declaration.py | 36 ++++++++++++++++--- languages/python/tests/_declaration_cases.py | 20 +++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/languages/python/python/stackencrypt/_declaration.py b/languages/python/python/stackencrypt/_declaration.py index ced4b7df1..dac858fc7 100644 --- a/languages/python/python/stackencrypt/_declaration.py +++ b/languages/python/python/stackencrypt/_declaration.py @@ -424,7 +424,17 @@ def _is_class_level(hint: Any) -> bool: def _hints_unresolved(cls: type, error: Exception) -> DeclarationError: - field = _unresolved_field(cls) + unresolved = _unresolved_field(cls) + field = unresolved[1] if unresolved else None + if unresolved is not None and _is_bare_class_var(*unresolved): + return DeclarationError( + "declaration_hints_unresolved", + f"field {field!r} is a bare ClassVar written as a string, which " + "this version of Python cannot read. Give it a type: " + "ClassVar[int], not ClassVar.", + field=field, + reason="bare_class_var", + ) where = f"field {field!r}" if field else f"a type hint of class {cls.__name__!r}" return DeclarationError( "declaration_hints_unresolved", @@ -435,8 +445,9 @@ def _hints_unresolved(cls: type, error: Exception) -> DeclarationError: ) -def _unresolved_field(cls: type) -> Optional[str]: - """The first field whose hint does not resolve, or ``None``. +def _unresolved_field(cls: type) -> Optional[Tuple[type, str, Any]]: + """The first field whose hint does not resolve, as the class that + declares it, its name and its hint, or ``None``. The hints are resolved one at a time, in the order ``get_type_hints`` resolves them, with the same namespaces. On 3.14, reading a class's @@ -451,10 +462,27 @@ def _unresolved_field(cls: type) -> Optional[str]: try: _resolve_one(base, name, hint) except _UNRESOLVED: - return name + return base, name, hint return None +def _is_bare_class_var(base: type, name: str, hint: Any) -> bool: + """Whether ``hint`` is a string that names ``ClassVar`` with no type. + + Python 3.9 cannot read one, as the future import writes it. The string is + evaluated as ``get_type_hints`` evaluates it, so ``typing.ClassVar`` or an + alias of it is found too. + """ + if not isinstance(hint, str): + return False + module = sys.modules.get(base.__module__) + global_names = dict(vars(module)) if module is not None else {} + try: + return eval(hint, global_names, dict(vars(base))) is ClassVar # noqa: S307 + except Exception: # noqa: BLE001 Any other failure is not this case. + return False + + def _own_annotations(base: type) -> Dict[str, Any]: """The annotations written on ``base`` itself, not on its bases.""" if sys.version_info < (3, 10): diff --git a/languages/python/tests/_declaration_cases.py b/languages/python/tests/_declaration_cases.py index 82a533354..44f8a0b87 100644 --- a/languages/python/tests/_declaration_cases.py +++ b/languages/python/tests/_declaration_cases.py @@ -12,6 +12,7 @@ import pickle import re import sys +import typing import uuid from typing import Annotated, ClassVar, Final, List, NamedTuple, Optional, Union @@ -496,6 +497,25 @@ class User(KeywordConstructor): assert planned(User) == [("email", TextEq)] +def test_a_bare_class_var_python_cannot_read_is_refused_with_advice(): + # Python 3.9 cannot read a bare ClassVar written as a string, as the + # future import writes it. Python 3.14 can. The test asks this Python. + class User(KeywordConstructor): + email: Annotated[str, TextEq] + count: ClassVar = 0 + + try: + typing.get_type_hints(User, include_extras=True) + except TypeError: + error = refusal(lambda: stackencrypt.model("users")(User)) + assert error.code == "declaration_hints_unresolved" + assert error.field == "count" + assert error.reason == "bare_class_var" + assert "ClassVar[int]" in str(error) + else: + assert planned(stackencrypt.model("users")(User)) == [("email", TextEq)] + + class RefusedBase(DeclarativeBase): pass From c7d6eb1b7ac5351cc2ec5925ee98cce0b1225931 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 16:52:15 +0900 Subject: [PATCH 07/21] fix(python): build a new engine client in a forked child The engine client includes a tokio runtime with its own threads. fork() copies only the thread that called it, so a child that Gunicorn or multiprocessing forked after the first call held a client whose runtime had no threads. Its next call never returned. The client is now stored with the ID of the process that built it. A child builds its own client on its first call, and loads the keyset again. The parent's copy is never dropped in the child: dropping a runtime whose threads are gone could block. macOS does not allow its TLS certificate service in a child that fork made, so on macOS the child crashes when it builds its client. The test skips on macOS, and test:live leaves it out there. CI runs it on Linux. run! also had a label that only the test build used. clippy with -D warnings refused it in the normal build. It is allowed now. Addresses review thread 1499-F4. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/crates/core/src/client.rs | 51 +++++++++++++++---- languages/python/mise.toml | 9 +++- .../python/python/stackencrypt/_cipher.py | 6 +++ languages/python/tests/_fork.py | 50 ++++++++++++++++++ .../python/tests/test_concurrency_live.py | 14 +++++ 5 files changed, 118 insertions(+), 12 deletions(-) create mode 100644 languages/python/tests/_fork.py diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index a86d59dca..1af788760 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -3,7 +3,9 @@ //! Every `Cipher` in the process uses the same `StackCipher`. It is built on //! the first operation that needs it, by whichever thread gets there first, //! and never rebuilt: credentials are read once, and the default keyset is -//! loaded once (the spec in #1341, "Operations"). Beside it is one +//! loaded once (the spec in #1341, "Operations"). The one exception is a +//! child that `fork` made: it builds a client of its own on its first call, +//! as described at [`shared`]. Beside it is one //! multi-thread tokio runtime. An operation hands its future to that runtime //! and blocks its own thread, with the GIL released, so calls from several //! Python threads run inside the engine at the same time. @@ -16,7 +18,9 @@ use std::collections::BTreeMap; use std::future::Future; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Mutex, MutexGuard, OnceLock}; +#[cfg(feature = "deterministic-kms")] +use std::sync::OnceLock; +use std::sync::{Mutex, MutexGuard}; use stack_auth::AutoStrategy; use stack_encrypt::kms::{ @@ -39,7 +43,11 @@ pub struct Client { runtime: tokio::runtime::Runtime, } -static CLIENT: OnceLock = OnceLock::new(); +/// The client, with the ID of the process that built it. A child that +/// `fork` made holds a copy of its parent's client, and that copy's runtime +/// has no threads: a fork copies only the thread that called it. So a client +/// is used only in the process that built it. +static CLIENT: Mutex> = Mutex::new(None); // Serialises the build, so two threads making their first call at once build // one client and load the keyset once. `OnceLock::get_or_init` would do the // same, but it cannot fail, and a failed build must not be cached: the user @@ -58,8 +66,22 @@ fn locked(mutex: &Mutex) -> MutexGuard<'_, T> { .unwrap_or_else(|poisoned| poisoned.into_inner()) } +/// This process's client, if it has built one. +fn current() -> Option<&'static Client> { + match *locked(&CLIENT) { + Some((pid, client)) if pid == std::process::id() => Some(client), + _ => None, + } +} + /// The shared client, built on the first call. /// +/// A child that `fork` made after its parent built the client builds a new +/// one on its first call, and loads the keyset again. The parent's copy is +/// never dropped in the child: dropping a runtime whose threads are gone +/// could block. A fork while another thread is building the client is not +/// safe: the child's copy of the build lock stays locked. +/// /// # Errors /// /// The refusal of whatever failed: the credentials (`stack_auth_*`), the @@ -68,7 +90,7 @@ fn locked(mutex: &Mutex) -> MutexGuard<'_, T> { pub fn shared() -> Result<&'static Client, Refusal> { #[cfg(feature = "deterministic-kms")] refuse_after_switch()?; - if let Some(client) = CLIENT.get() { + if let Some(client) = current() { return Ok(client); } let _building = locked(&BUILDING); @@ -76,12 +98,15 @@ pub fn shared() -> Result<&'static Client, Refusal> { // it, and the live client must then not be built. #[cfg(feature = "deterministic-kms")] refuse_after_switch()?; - if let Some(client) = CLIENT.get() { + if let Some(client) = current() { return Ok(client); } - let client = build()?; + // Leaked on purpose: a client lives as long as its process, and a copy + // inherited through fork must never be dropped. + let client: &'static Client = Box::leak(Box::new(build()?)); BUILDS.fetch_add(1, Ordering::Relaxed); - Ok(CLIENT.get_or_init(|| client)) + *locked(&CLIENT) = Some((std::process::id(), client)); + Ok(client) } /// Run an operation's future on the process's client, and return its result. @@ -97,7 +122,9 @@ pub fn shared() -> Result<&'static Client, Refusal> { /// not the function, so code after `run!` runs in the test build too. macro_rules! run { (|$cipher:ident| $future:expr) => {{ - 'run: { + // The label is used only in the test build. + #[allow(unused_labels)] + let value = 'run: { #[cfg(feature = "deterministic-kms")] if let Some((cipher, runtime)) = $crate::client::deterministic() { let $cipher = cipher; @@ -106,7 +133,8 @@ macro_rules! run { let client = $crate::client::shared()?; let $cipher = client.cipher(); client.block_on($future) - } + }; + value }}; } pub(crate) use run; @@ -160,7 +188,7 @@ pub fn use_deterministic(seed: [u8; 32]) -> Result<(), Refusal> { // The lock `shared` builds under: a live build in flight on another // thread finishes before this checks, so the two never both exist. let _building = locked(&BUILDING); - if CLIENT.get().is_some() || DETERMINISTIC.get().is_some() { + if locked(&CLIENT).is_some() || DETERMINISTIC.get().is_some() { return Err(refused( "the process already has a client; call this before any operation".to_owned(), )); @@ -188,7 +216,8 @@ pub fn deterministic() -> Option<&'static Deterministic> { } /// How many times the shared client has been built in this process: 0 -/// before the first operation, 1 after it. For the tests. +/// before the first operation, 1 after it. A forked child starts from its +/// parent's count, so its own build makes it 2. For the tests. pub fn build_count() -> u64 { BUILDS.load(Ordering::Relaxed) } diff --git a/languages/python/mise.toml b/languages/python/mise.toml index f0876d8fa..5ac478e7b 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -105,7 +105,14 @@ set -euo pipefail mise -C ../../packages/eql run build psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f ../../packages/eql/release/cipherstash-encrypt-v4.sql log="${TMPDIR:-/tmp}/stackencrypt-live.log" -.venv/bin/python -m pytest -m live -v -rs | tee "$log" +# macOS refuses its TLS certificate service in a child that fork made, so the +# fork test cannot run there. It is left out, not skipped, because a skip +# fails this task. CI runs on Linux, where it runs. +deselect=() +if [ "$(uname)" = Darwin ]; then + deselect=(--deselect tests/test_concurrency_live.py::test_a_forked_child_builds_its_own_client_and_its_calls_succeed) +fi +.venv/bin/python -m pytest -m live -v -rs "${deselect[@]}" | tee "$log" if grep -q ' SKIPPED' "$log"; then echo "error: a live test skipped; set the four CS_* credentials and EQL_TEST_DATABASE_URL" >&2 exit 1 diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index bebc3415d..146dbc20f 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -52,6 +52,12 @@ class Cipher(Generic[T]): Every cipher in the process shares one engine client. It is built on the first ``encrypt`` or ``decrypt`` of any cipher, which loads the default keyset once. + + A child process that ``os.fork`` made, as Gunicorn's workers are, builds + a client of its own on its first call, and loads the keyset again. On + macOS, the child crashes when it does: macOS does not allow its TLS + certificate service after a fork. Build no client before the fork there, + or start child processes with ``spawn``. """ def __init__(self, cls: Type[T]) -> None: diff --git a/languages/python/tests/_fork.py b/languages/python/tests/_fork.py new file mode 100644 index 000000000..13898a1d9 --- /dev/null +++ b/languages/python/tests/_fork.py @@ -0,0 +1,50 @@ +"""Child process of `test_concurrency_live.py`: one call, then a fork, then a +call in the forked child. Prints what the child did as JSON. + +The child's result comes back through a pipe. The parent waits at most 30 +seconds, so a child that never returns shows as "hang" and does not stop +the test run. +""" + +import json +import os +import time + +from _sync_contract import User, user + +from stackencrypt import Cipher, _engine + +cipher = Cipher(User) +cipher.encrypt([user(1)]) +assert _engine.client_builds() == 1 + +read_end, write_end = os.pipe() +pid = os.fork() +if pid == 0: + os.close(read_end) + try: + rows = cipher.encrypt([user(2)]) + opened = cipher.decrypt(rows) + result = { + "child": "ok" if opened == [user(2)] else "wrong", + "builds": _engine.client_builds(), + } + except BaseException as error: # noqa: BLE001 Reported to the parent. + result = {"child": f"error: {type(error).__name__}"} + os.write(write_end, json.dumps(result).encode()) + os._exit(0) + +os.close(write_end) +deadline = time.monotonic() + 30 +while time.monotonic() < deadline: + done, status = os.waitpid(pid, os.WNOHANG) + if done: + break + time.sleep(0.05) +else: + os.kill(pid, 9) + os.waitpid(pid, 0) + print(json.dumps({"child": "hang"})) + raise SystemExit(0) + +print(os.read(read_end, 4096).decode() or json.dumps({"child": f"no result, status {status}"})) diff --git a/languages/python/tests/test_concurrency_live.py b/languages/python/tests/test_concurrency_live.py index 2127fe04a..7a373c8ef 100644 --- a/languages/python/tests/test_concurrency_live.py +++ b/languages/python/tests/test_concurrency_live.py @@ -48,6 +48,20 @@ def test_threads_decrypt_at_once_and_every_row_opens(): assert decrypt_from_threads(cipher, rows) == [user(n) for n in range(THREADS * PER_THREAD)] +@pytest.mark.skipif(sys.platform == "win32", reason="Windows has no fork") +@pytest.mark.skipif( + sys.platform == "darwin", + reason="macOS refuses its TLS certificate service in a child that fork made", +) +def test_a_forked_child_builds_its_own_client_and_its_calls_succeed(): + # The child's copy of the client has a runtime with no threads: a fork + # copies only the thread that called it. The child builds a client of + # its own on its first call. + result = run_python([str(HERE / "_fork.py")]) + + assert result == {"child": "ok", "builds": 2}, result + + @pytest.mark.skipif(sys.platform == "win32", reason="SIGINT is sent with os.kill") def test_an_interrupt_during_encrypt_waits_for_it_and_the_cipher_still_works(): result = run_python([str(HERE / "_interrupt.py")]) From 3c2b14ea9e591fbc61ed3f40e76308a617b4ef5b Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 17:33:59 +0900 Subject: [PATCH 08/21] fix(python): refuse a forked child the client its parent built The engine client includes a tokio runtime with its own threads. fork() copies only the thread that called it, so a child forked after the first call held a client whose runtime had no threads. Its next call never returned. Building a new runtime in the child is not safe either. tokio's locks keep a process-wide table of waiting threads, and the child's copy lists the parent's threads. On Linux, a new runtime in the child blocked in 8 of 15 runs. So a child that fork made after the client was built is now refused, with StashError code stack_forked. The message says to make the first call after the fork. The check reads a OnceLock and an atomic process ID, and takes no lock, so it cannot block. A child forked before any call builds its own client, as before. The fork test covers both cases, and runs on macOS and Linux. On Linux, each case passed 15 of 15 runs. Addresses review thread 1499-F4. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/crates/core/src/client.rs | 82 +++++++++++-------- languages/python/mise.toml | 9 +- .../python/python/stackencrypt/_cipher.py | 12 +-- languages/python/tests/_fork.py | 24 ++++-- .../python/tests/test_concurrency_live.py | 23 +++--- 5 files changed, 82 insertions(+), 68 deletions(-) diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index 1af788760..9958eeadf 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -3,9 +3,9 @@ //! Every `Cipher` in the process uses the same `StackCipher`. It is built on //! the first operation that needs it, by whichever thread gets there first, //! and never rebuilt: credentials are read once, and the default keyset is -//! loaded once (the spec in #1341, "Operations"). The one exception is a -//! child that `fork` made: it builds a client of its own on its first call, -//! as described at [`shared`]. Beside it is one +//! loaded once (the spec in #1341, "Operations"). A child that `fork` made +//! after the client was built cannot use it, and is refused, as described at +//! [`shared`]. Beside it is one //! multi-thread tokio runtime. An operation hands its future to that runtime //! and blocks its own thread, with the GIL released, so calls from several //! Python threads run inside the engine at the same time. @@ -17,10 +17,8 @@ use std::collections::BTreeMap; use std::future::Future; -use std::sync::atomic::{AtomicU64, Ordering}; -#[cfg(feature = "deterministic-kms")] -use std::sync::OnceLock; -use std::sync::{Mutex, MutexGuard}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::{Mutex, MutexGuard, OnceLock}; use stack_auth::AutoStrategy; use stack_encrypt::kms::{ @@ -43,11 +41,10 @@ pub struct Client { runtime: tokio::runtime::Runtime, } -/// The client, with the ID of the process that built it. A child that -/// `fork` made holds a copy of its parent's client, and that copy's runtime -/// has no threads: a fork copies only the thread that called it. So a client -/// is used only in the process that built it. -static CLIENT: Mutex> = Mutex::new(None); +static CLIENT: OnceLock = OnceLock::new(); +/// The ID of the process that built `CLIENT`. Set before `CLIENT`, so a +/// thread that sees the client sees its process ID. +static BUILT_IN: AtomicU32 = AtomicU32::new(0); // Serialises the build, so two threads making their first call at once build // one client and load the keyset once. `OnceLock::get_or_init` would do the // same, but it cannot fail, and a failed build must not be cached: the user @@ -66,31 +63,47 @@ fn locked(mutex: &Mutex) -> MutexGuard<'_, T> { .unwrap_or_else(|poisoned| poisoned.into_inner()) } -/// This process's client, if it has built one. -fn current() -> Option<&'static Client> { - match *locked(&CLIENT) { - Some((pid, client)) if pid == std::process::id() => Some(client), - _ => None, +/// The client, if this process built it. A refusal if another process +/// built it: this process is a child that `fork` made after the build. +/// +/// The child's copy of the client cannot work: its runtime has no threads, +/// because a fork copies only the thread that called it. A new runtime is +/// not safe in the child either: tokio's locks keep a process-wide table of +/// waiting threads, and the child's copy lists the parent's threads. A new +/// runtime there blocked in 8 of 15 runs on Linux. So the child is refused. +/// No lock is taken here, so the refusal cannot block. +fn current() -> Result, Refusal> { + let Some(client) = CLIENT.get() else { + return Ok(None); + }; + if BUILT_IN.load(Ordering::Acquire) == std::process::id() { + return Ok(Some(client)); } + Err(Refusal { + code: "stack_forked".to_owned(), + message: "this process was forked after the engine client was built, and a forked \ + process cannot use it. Make the first encrypt, decrypt or query call after \ + the fork: with Gunicorn, do not use --preload; with multiprocessing, use \ + the spawn start method." + .to_owned(), + field: None, + reason: None, + }) } /// The shared client, built on the first call. /// -/// A child that `fork` made after its parent built the client builds a new -/// one on its first call, and loads the keyset again. The parent's copy is -/// never dropped in the child: dropping a runtime whose threads are gone -/// could block. A fork while another thread is building the client is not -/// safe: the child's copy of the build lock stays locked. -/// /// # Errors /// -/// The refusal of whatever failed: the credentials (`stack_auth_*`), the -/// client key or the ZeroKMS client (`stack_kms_*`), or the keyset load -/// (`stack_encrypt_*`). Nothing is cached on failure. +/// `stack_forked` in a child that `fork` made after the client was built +/// (see [`current`]). Otherwise the refusal of whatever failed: the +/// credentials (`stack_auth_*`), the client key or the ZeroKMS client +/// (`stack_kms_*`), or the keyset load (`stack_encrypt_*`). Nothing is +/// cached on failure. pub fn shared() -> Result<&'static Client, Refusal> { #[cfg(feature = "deterministic-kms")] refuse_after_switch()?; - if let Some(client) = current() { + if let Some(client) = current()? { return Ok(client); } let _building = locked(&BUILDING); @@ -98,15 +111,13 @@ pub fn shared() -> Result<&'static Client, Refusal> { // it, and the live client must then not be built. #[cfg(feature = "deterministic-kms")] refuse_after_switch()?; - if let Some(client) = current() { + if let Some(client) = current()? { return Ok(client); } - // Leaked on purpose: a client lives as long as its process, and a copy - // inherited through fork must never be dropped. - let client: &'static Client = Box::leak(Box::new(build()?)); + let client = build()?; BUILDS.fetch_add(1, Ordering::Relaxed); - *locked(&CLIENT) = Some((std::process::id(), client)); - Ok(client) + BUILT_IN.store(std::process::id(), Ordering::Release); + Ok(CLIENT.get_or_init(|| client)) } /// Run an operation's future on the process's client, and return its result. @@ -188,7 +199,7 @@ pub fn use_deterministic(seed: [u8; 32]) -> Result<(), Refusal> { // The lock `shared` builds under: a live build in flight on another // thread finishes before this checks, so the two never both exist. let _building = locked(&BUILDING); - if locked(&CLIENT).is_some() || DETERMINISTIC.get().is_some() { + if CLIENT.get().is_some() || DETERMINISTIC.get().is_some() { return Err(refused( "the process already has a client; call this before any operation".to_owned(), )); @@ -216,8 +227,7 @@ pub fn deterministic() -> Option<&'static Deterministic> { } /// How many times the shared client has been built in this process: 0 -/// before the first operation, 1 after it. A forked child starts from its -/// parent's count, so its own build makes it 2. For the tests. +/// before the first operation, 1 after it. For the tests. pub fn build_count() -> u64 { BUILDS.load(Ordering::Relaxed) } diff --git a/languages/python/mise.toml b/languages/python/mise.toml index 5ac478e7b..f0876d8fa 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -105,14 +105,7 @@ set -euo pipefail mise -C ../../packages/eql run build psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f ../../packages/eql/release/cipherstash-encrypt-v4.sql log="${TMPDIR:-/tmp}/stackencrypt-live.log" -# macOS refuses its TLS certificate service in a child that fork made, so the -# fork test cannot run there. It is left out, not skipped, because a skip -# fails this task. CI runs on Linux, where it runs. -deselect=() -if [ "$(uname)" = Darwin ]; then - deselect=(--deselect tests/test_concurrency_live.py::test_a_forked_child_builds_its_own_client_and_its_calls_succeed) -fi -.venv/bin/python -m pytest -m live -v -rs "${deselect[@]}" | tee "$log" +.venv/bin/python -m pytest -m live -v -rs | tee "$log" if grep -q ' SKIPPED' "$log"; then echo "error: a live test skipped; set the four CS_* credentials and EQL_TEST_DATABASE_URL" >&2 exit 1 diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index 146dbc20f..996a55490 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -53,11 +53,13 @@ class Cipher(Generic[T]): first ``encrypt`` or ``decrypt`` of any cipher, which loads the default keyset once. - A child process that ``os.fork`` made, as Gunicorn's workers are, builds - a client of its own on its first call, and loads the keyset again. On - macOS, the child crashes when it does: macOS does not allow its TLS - certificate service after a fork. Build no client before the fork there, - or start child processes with ``spawn``. + A process that ``os.fork`` made, as Gunicorn's workers are, cannot use a + client its parent built: the first ``encrypt``, ``decrypt`` or query + call must come after the fork. A call in such a child raises + ``StashError`` with the code ``stack_forked``. A fork before the first + call is fine: each child builds its own client. With Gunicorn, do not + use ``--preload``. With ``multiprocessing``, use the ``spawn`` start + method. """ def __init__(self, cls: Type[T]) -> None: diff --git a/languages/python/tests/_fork.py b/languages/python/tests/_fork.py index 13898a1d9..0720c06e2 100644 --- a/languages/python/tests/_fork.py +++ b/languages/python/tests/_fork.py @@ -1,5 +1,9 @@ -"""Child process of `test_concurrency_live.py`: one call, then a fork, then a -call in the forked child. Prints what the child did as JSON. +"""Child process of `test_concurrency_live.py`: a fork, and a call in the +forked child. Prints what the child did as JSON. + +`after` forks after the first call, so the parent's client exists: the +child must be refused, with no wait. `before` forks before any call, so no +client exists: the child must build its own and work. The child's result comes back through a pipe. The parent waits at most 30 seconds, so a child that never returns shows as "hang" and does not stop @@ -8,15 +12,18 @@ import json import os +import sys import time from _sync_contract import User, user -from stackencrypt import Cipher, _engine +from stackencrypt import Cipher, StashError, _engine +mode = sys.argv[1] cipher = Cipher(User) -cipher.encrypt([user(1)]) -assert _engine.client_builds() == 1 +if mode == "after": + cipher.encrypt([user(1)]) + assert _engine.client_builds() == 1 read_end, write_end = os.pipe() pid = os.fork() @@ -25,10 +32,9 @@ try: rows = cipher.encrypt([user(2)]) opened = cipher.decrypt(rows) - result = { - "child": "ok" if opened == [user(2)] else "wrong", - "builds": _engine.client_builds(), - } + result = {"child": "ok" if opened == [user(2)] else "wrong"} + except StashError as error: + result = {"child": error.code} except BaseException as error: # noqa: BLE001 Reported to the parent. result = {"child": f"error: {type(error).__name__}"} os.write(write_end, json.dumps(result).encode()) diff --git a/languages/python/tests/test_concurrency_live.py b/languages/python/tests/test_concurrency_live.py index 7a373c8ef..1fcb16fb6 100644 --- a/languages/python/tests/test_concurrency_live.py +++ b/languages/python/tests/test_concurrency_live.py @@ -49,17 +49,20 @@ def test_threads_decrypt_at_once_and_every_row_opens(): @pytest.mark.skipif(sys.platform == "win32", reason="Windows has no fork") -@pytest.mark.skipif( - sys.platform == "darwin", - reason="macOS refuses its TLS certificate service in a child that fork made", +@pytest.mark.parametrize( + "mode, child", + [ + # The parent's client cannot work in the child, and a new runtime + # there can block. The child is refused at once, with what to do. + ("after", "stack_forked"), + # No client existed at the fork, so the child builds its own. + ("before", "ok"), + ], ) -def test_a_forked_child_builds_its_own_client_and_its_calls_succeed(): - # The child's copy of the client has a runtime with no threads: a fork - # copies only the thread that called it. The child builds a client of - # its own on its first call. - result = run_python([str(HERE / "_fork.py")]) - - assert result == {"child": "ok", "builds": 2}, result +def test_a_forked_child_is_refused_only_when_the_client_was_built_before_the_fork( + mode, child +): + assert run_python([str(HERE / "_fork.py"), mode]) == {"child": child} @pytest.mark.skipif(sys.platform == "win32", reason="SIGINT is sent with os.kill") From 7c7a534f1ea621b41cf41dc6b7c9266b62c9f70a Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 18:02:52 +0900 Subject: [PATCH 09/21] fix(python): resolve the root workspace's vitaminc, and test that it stays so AGENTS.md said the Python Cargo.lock is seeded from the root one, so the crates the two workspaces share resolve to the versions the root tests. That was no longer true: the Python lock had vitaminc 0.8.1, and the root lock had 0.8.0. No check compared the two locks. The Python lock is seeded from the root lock again. The difference was small: 0.8.1 adds a storage profile to vitaminc-aead-frame. The SDK does not use it. The encryption and order crates have the same source in both versions. The documented steps were not enough to re-seed: cargo metadata moved vitaminc back to 0.8.1, and cargo update --precise moved each crate to 0.8.0. AGENTS.md now gives that step. python-lock-parity.test.mjs compares the crates that decide what a stored value is, by version, source and checksum: every vitaminc crate, recipher, ore-rs, clww-ore, cts-common and zerokms-protocol. Other shared crates may differ. The lock parser moves to lib/cargo-lock.mjs, so this test and the cipherstash-client one share it. The deterministic suite still opens the Go SDK's frozen rows and seals the same values, so vitaminc 0.8.0 gives the same results here. Addresses review thread 1406-F4. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- languages/python/Cargo.lock | 297 +++++++++++------- .../cipherstash-client-parity.test.mjs | 23 +- scripts/__tests__/lib/cargo-lock.mjs | 22 ++ scripts/__tests__/python-lock-parity.test.mjs | 83 +++++ 5 files changed, 297 insertions(+), 130 deletions(-) create mode 100644 scripts/__tests__/python-lock-parity.test.mjs diff --git a/AGENTS.md b/AGENTS.md index 8aa75cf54..3fe4f70e6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin, never in a crate manifest, so `cargo test` links libpython. `pyproject.toml` sets it, and `test:deterministic` names it again in its `--features` list; `scripts/__tests__/python-ci.test.mjs` holds the two lists together. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It also runs one test binary of `crates/core`, `deterministic_switch`, with the `deterministic-kms` feature: the switch is for the whole process, so that test needs a process of its own. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. A third job, `wheels`, builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin, never in a crate manifest, so `cargo test` links libpython. `pyproject.toml` sets it, and `test:deterministic` names it again in its `--features` list; `scripts/__tests__/python-ci.test.mjs` holds the two lists together. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It also runs one test binary of `crates/core`, `deterministic_switch`, with the `deterministic-kms` feature: the switch is for the whole process, so that test needs a process of its own. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so the crates that decide what a stored value is resolve to the versions the root tests: every `vitaminc` crate, `recipher`, `ore-rs`, `clww-ore`, `cts-common` and `zerokms-protocol`. `scripts/__tests__/python-lock-parity.test.mjs` compares those crates in the two locks, by version, source and checksum. Other shared crates may differ. To re-seed, copy the root `Cargo.lock` over it and run `cargo metadata` there. That command prunes the copy to what the workspace uses. It can also move a crate to a newer version, when this workspace turns on a feature the root does not. Move each such crate back with `cargo update -p @ --precise `, until the parity test passes. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. A third job, `wheels`, builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `languages/python/examples/customer-directory`: An internal FastAPI example of the Python SDK, written twice: with psycopg (`app.py`) and with SQLAlchemy (`app_sqlalchemy.py`). It has its own `pyproject.toml` and `uv.lock`, and installs `stackencrypt` from this checkout. Its tests need real credentials, and CI does not run them. diff --git a/languages/python/Cargo.lock b/languages/python/Cargo.lock index f1746dd78..6a522d95f 100644 --- a/languages/python/Cargo.lock +++ b/languages/python/Cargo.lock @@ -412,9 +412,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.43" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "num-traits", @@ -455,7 +455,7 @@ dependencies = [ "subtle", "thiserror 1.0.69", "vitaminc-ore", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", "zeroize", ] @@ -1351,7 +1351,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", "tokio", "tower-service", "tracing", @@ -1537,15 +1537,14 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.3.4" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2", + "socket2 0.5.10", "widestring", - "windows-registry", - "windows-result", - "windows-sys 0.61.2", + "windows-sys 0.48.0", + "winreg", ] [[package]] @@ -1557,6 +1556,16 @@ dependencies = [ "serde", ] +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -1702,9 +1711,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "leb128fmt" @@ -1982,12 +1991,12 @@ dependencies = [ "rand 0.10.1", "subtle-ng", "thiserror 1.0.69", - "vitaminc-context 0.8.1", + "vitaminc-context 0.8.0", "vitaminc-hmac", "vitaminc-ore", "vitaminc-prf", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] @@ -2236,7 +2245,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -2275,7 +2284,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] @@ -2444,22 +2453,22 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 3.0.6", ] [[package]] @@ -2752,9 +2761,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.17.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ "core-foundation-sys", "libc", @@ -2962,6 +2971,16 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -3003,8 +3022,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc 0.8.1", - "vitaminc-protected 0.8.1", + "vitaminc 0.8.0", + "vitaminc-protected 0.8.0", "web-time", "zeroize", "zerokms-protocol", @@ -3030,15 +3049,15 @@ dependencies = [ "stack-kms", "thiserror 1.0.69", "uuid", - "vitaminc 0.8.1", - "vitaminc-aead 0.8.1", + "vitaminc 0.8.0", + "vitaminc-aead 0.8.0", "vitaminc-aead-frame", - "vitaminc-encrypt 0.8.1", + "vitaminc-encrypt 0.8.0", "vitaminc-hmac", "vitaminc-normalize", "vitaminc-ore", "vitaminc-prf", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", "zeroize", ] @@ -3076,8 +3095,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc 0.8.1", - "vitaminc-protected 0.8.1", + "vitaminc 0.8.0", + "vitaminc-protected 0.8.0", "zeroize", "zerokms-protocol", ] @@ -3345,7 +3364,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", "windows-sys 0.61.2", ] @@ -3431,9 +3450,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.11" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", "bitflags", @@ -3443,13 +3462,13 @@ dependencies = [ "http", "http-body", "http-body-util", + "iri-string", "pin-project-lite", "tokio", "tokio-util", "tower", "tower-layer", "tower-service", - "url", ] [[package]] @@ -3709,16 +3728,16 @@ dependencies = [ [[package]] name = "vitaminc" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd347492314ce981cd9fba0361e07d240e4e4a2e12e0107a7b86cd16abb9f8aa" +checksum = "ea37168a603d605732d0bf0672586139ab91736f767d7b85e173c84dcee19054" dependencies = [ - "vitaminc-aead 0.8.1", - "vitaminc-context 0.8.1", - "vitaminc-encrypt 0.8.1", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", - "vitaminc-traits 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-context 0.8.0", + "vitaminc-encrypt 0.8.0", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", + "vitaminc-traits 0.8.0", ] [[package]] @@ -3738,18 +3757,18 @@ dependencies = [ [[package]] name = "vitaminc-aead" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e723c592529da633fe7ba94d95904a9b6832e7995fc3f1d0da17bdbf8587c7a1" +checksum = "e9bb97fd543ee6a2b168e257ec69d0d274a775b1eaa3f856b5ac2941b01891c1" dependencies = [ "bytes", "futures-util", "serde", "thiserror 2.0.18", - "vitaminc-aead-derive 0.8.1", - "vitaminc-context 0.8.1", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", + "vitaminc-aead-derive 0.8.0", + "vitaminc-context 0.8.0", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] @@ -3766,9 +3785,9 @@ dependencies = [ [[package]] name = "vitaminc-aead-derive" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "553187b663af2a88f50cddc6fc4b5e58f045aa5f7296075cf33a56c972ff6b2b" +checksum = "0845a44c07ddc882ed47a0e92e04fc83496220f054b589ad0cbd215c5009e5b7" dependencies = [ "proc-macro2", "quote", @@ -3777,13 +3796,12 @@ dependencies = [ [[package]] name = "vitaminc-aead-frame" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a65349bb75a9ecc9fd4f057e5a6aac964bea68e57d03d55118a22455002a424" +checksum = "3b0299777255171712a296d59164691dc1e93d10bddaab6e248fede1790dfd2e" dependencies = [ "serde", - "vitaminc-aead 0.8.1", - "vitaminc-protected 0.8.1", + "vitaminc-aead 0.8.0", "zeroize", ] @@ -3799,12 +3817,12 @@ dependencies = [ [[package]] name = "vitaminc-context" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0179df09c5d8cab1c9d9f104eb807812851d1a36f97212542fb61f29e01646e4" +checksum = "a630079ba2d50680ee5048addd430eda30613559809dd6d383551e6d3d299826" dependencies = [ "mutants", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", ] [[package]] @@ -3823,72 +3841,72 @@ dependencies = [ [[package]] name = "vitaminc-encrypt" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265383e1184382f22570fa5be60db8b4f6b2321c21a2c5bf08a858705c4c8828" +checksum = "db45e073f92ae8ca27f8fe14270bc34c75e767559c2347098dacca95180d1ddd" dependencies = [ "aes-gcm", "aws-lc-rs", - "vitaminc-aead 0.8.1", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] [[package]] name = "vitaminc-hmac" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a244886bff4915e15000a69f5a10f14410cabb30c445e3d585057c06a9ce1a" +checksum = "b5fa0ae8bbad85ae30fb5c91b0b72b694d59bf2c2c1e7ad799eda66fdbd67e7c" dependencies = [ "hmac", "sha2 0.11.0", "vitaminc-prf", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", "zeroize", ] [[package]] name = "vitaminc-normalize" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d820d507ca7cce81772e38c1277b26acc422a4fd99402e6d9d49c412a5a135f0" +checksum = "f5a2d8875e0b8b35b7a4f9dd698bbd13c989837cea564b5a4cbb5371b78fa937" dependencies = [ "thiserror 2.0.18", "unicode-general-category", "unicode-normalization", - "vitaminc-aead 0.8.1", - "vitaminc-protected 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-protected 0.8.0", ] [[package]] name = "vitaminc-ore" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b80bf6789c1e2e1c192a255a9e2f6c3e7ffbce2fcb61d60edecee6ffc2f71ef" +checksum = "f05518b4d35cb0c592dad59afb11b8e883ea252449b9ae47ff5ddeff0ed49f75" dependencies = [ "thiserror 2.0.18", "unicode-general-category", "unicode-normalization", - "vitaminc-aead 0.8.1", - "vitaminc-context 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-context 0.8.0", "vitaminc-normalize", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", "zeroize", ] [[package]] name = "vitaminc-prf" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86cfcbfb4089f5bf334723cbf106f7d0d4709abf0ad12aaf49d6cf89f3e78de" +checksum = "916688478273ca3e5c6805b355534ce9263688391cd4ab23b08bef760039d325" dependencies = [ "mutants", "thiserror 2.0.18", - "vitaminc-aead 0.8.1", - "vitaminc-context 0.8.1", + "vitaminc-aead 0.8.0", + "vitaminc-context 0.8.0", "vitaminc-normalize", - "vitaminc-protected 0.8.1", + "vitaminc-protected 0.8.0", ] [[package]] @@ -3910,9 +3928,9 @@ dependencies = [ [[package]] name = "vitaminc-protected" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81704a2c5bb14d108dd4aa20b4124d4b0c47a2695fc6330131cf0897350737aa" +checksum = "a6edc22562a9bbff0ba9704a2c43670b2c7cb1637524c045c3ba6436ed00917f" dependencies = [ "bitvec", "digest 0.11.3", @@ -3921,7 +3939,7 @@ dependencies = [ "serde_bytes", "subtle", "thiserror 2.0.18", - "vitaminc-protected-derive 0.8.1", + "vitaminc-protected-derive 0.8.0", "zeroize", ] @@ -3938,9 +3956,9 @@ dependencies = [ [[package]] name = "vitaminc-protected-derive" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eacd4b9ad7ae5196ee117944325aead3508da45d4c7ad73f682e64d62b09bb7" +checksum = "a7cce3a7d0ce6f57c0e82345f20e87f3737cccd3a70aec2a2ab49f68a3a5e24a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -3965,16 +3983,16 @@ dependencies = [ [[package]] name = "vitaminc-random" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9bc57a6532bfa847fa5093c2c39bdc333d1c0b724f982cf3c240e8b091d491" +checksum = "8bb253f783ce6f31a55a92f5a9f0526109b8caa6a465edb94244fd344a078c11" dependencies = [ "chacha20", "getrandom 0.4.2", "rand 0.10.1", "thiserror 2.0.18", - "vitaminc-protected 0.8.1", - "vitaminc-random-derives 0.8.1", + "vitaminc-protected 0.8.0", + "vitaminc-random-derives 0.8.0", "zeroize", ] @@ -3991,9 +4009,9 @@ dependencies = [ [[package]] name = "vitaminc-random-derives" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73e700b3a8e01ee5130939fbdc4dadb38bc4d4160c5f1d5f7fd139bab0491dfa" +checksum = "df0b3615936d6dded4b775856f4b0fa229d745d183f0b3754ac80035f16c27d9" dependencies = [ "proc-macro2", "quote", @@ -4018,17 +4036,17 @@ dependencies = [ [[package]] name = "vitaminc-traits" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a090c4d967d90bc3c609b73a3629e27ca752e0a5a62a5b00e0ea91f04617c60d" +checksum = "1e9098ee4b13446ef959583ee2f3c384df571b453179104f7b4960ffddf65ee9" dependencies = [ "anyhow", "bytes", "rmp-serde", "serde", "thiserror 2.0.18", - "vitaminc-protected 0.8.1", - "vitaminc-random 0.8.1", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] @@ -4288,17 +4306,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -4326,6 +4333,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -4368,6 +4384,21 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -4407,6 +4438,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -4425,6 +4462,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -4443,6 +4486,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -4473,6 +4522,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -4491,6 +4546,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -4509,6 +4570,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -4527,6 +4594,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -4548,6 +4621,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/scripts/__tests__/cipherstash-client-parity.test.mjs b/scripts/__tests__/cipherstash-client-parity.test.mjs index 305ab4f65..7bc90cf2e 100644 --- a/scripts/__tests__/cipherstash-client-parity.test.mjs +++ b/scripts/__tests__/cipherstash-client-parity.test.mjs @@ -1,6 +1,7 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' +import { lockedPackages } from './lib/cargo-lock.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' /** @@ -32,28 +33,6 @@ import { REPO_ROOT } from './lib/repo-root.mjs' const EQL_LOCK = 'packages/eql/Cargo.lock' const FFI_LOCK = 'languages/typescript/packages/protect-ffi/Cargo.lock' -/** - * Every `name` package a `Cargo.lock` records, as its version, source and - * checksum, sorted by version. The version alone is not enough: a git source - * or a `[patch.crates-io]` fork can keep the version number while resolving - * different code, and the source and checksum are what tell them apart. - */ -export function lockedPackages(source, name) { - const field = (block, key) => - new RegExp(`^${key} = "([^"]*)"$`, 'm').exec(block)?.[1] ?? null - return source - .split(/^\[\[package\]\]$/m) - .slice(1) - .filter((block) => new RegExp(`^name = "${name}"$`, 'm').test(block)) - .map((block) => ({ - version: field(block, 'version'), - source: field(block, 'source'), - checksum: field(block, 'checksum'), - })) - .filter((pkg) => pkg.version) - .sort((a, b) => a.version.localeCompare(b.version)) -} - const read = (file) => readFileSync(join(REPO_ROOT, file), 'utf8') describe('lockedPackages', () => { diff --git a/scripts/__tests__/lib/cargo-lock.mjs b/scripts/__tests__/lib/cargo-lock.mjs index e056c2ffe..fcce47d16 100644 --- a/scripts/__tests__/lib/cargo-lock.mjs +++ b/scripts/__tests__/lib/cargo-lock.mjs @@ -15,3 +15,25 @@ export function pathPackageNames(lock) { return name }) } + +/** + * Every `name` package a `Cargo.lock` records, as its version, source and + * checksum, sorted by version. The version alone is not enough: a git source + * or a `[patch.crates-io]` fork can keep the version number while resolving + * different code, and the source and checksum are what tell them apart. + */ +export function lockedPackages(source, name) { + const field = (block, key) => + new RegExp(`^${key} = "([^"]*)"$`, 'm').exec(block)?.[1] ?? null + return source + .split(/^\[\[package\]\]$/m) + .slice(1) + .filter((block) => new RegExp(`^name = "${name}"$`, 'm').test(block)) + .map((block) => ({ + version: field(block, 'version'), + source: field(block, 'source'), + checksum: field(block, 'checksum'), + })) + .filter((pkg) => pkg.version) + .sort((a, b) => a.version.localeCompare(b.version)) +} diff --git a/scripts/__tests__/python-lock-parity.test.mjs b/scripts/__tests__/python-lock-parity.test.mjs new file mode 100644 index 000000000..97eb63254 --- /dev/null +++ b/scripts/__tests__/python-lock-parity.test.mjs @@ -0,0 +1,83 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { lockedPackages } from './lib/cargo-lock.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' + +/** + * The Python SDK's workspace and the root workspace must resolve the same + * crates that decide what a stored value is: the encryption, the order + * terms, and the key protocol. + * + * `languages/python` is its own Cargo workspace, with its own Cargo.lock. Its + * engine builds the stack-* crates by path, but each lock picks its own + * versions of the crates from crates.io. The root workspace's tests run the + * stack crates against the root lock. If the Python lock resolves another + * version of one of these crates, the Python engine runs code those tests + * never ran, and it can write values the other SDKs do not read the same way. + * + * Each crate is compared by version, source and checksum, as + * `cipherstash-client-parity.test.mjs` compares its crates. Other shared + * crates, such as chrono, may differ. + * + * The fix for a failure is to seed the Python lock from the root one again, + * as AGENTS.md describes. + * + * Node builtins only: no cargo on `pnpm run test:scripts` (see AGENTS.md). + */ + +const ROOT_LOCK = 'Cargo.lock' +const PYTHON_LOCK = 'languages/python/Cargo.lock' + +const read = (file) => readFileSync(join(REPO_ROOT, file), 'utf8') +const root = read(ROOT_LOCK) +const python = read(PYTHON_LOCK) + +/** Every package name a lock records. */ +const names = (lock) => + new Set([...lock.matchAll(/^name = "([^"]+)"$/gm)].map((m) => m[1])) + +/** + * The crates that decide what a stored value is. Every vitaminc crate is + * one: vitaminc is the encryption and the order terms. The rest are named: + * recipher, and the two order schemes, which the root Cargo.toml pins with + * `=`; cts-common and zerokms-protocol, the key protocol. + */ +const VALUE_CRATES = [ + ...[...names(root)].filter((name) => name.startsWith('vitaminc')).sort(), + 'recipher', + 'ore-rs', + 'clww-ore', + 'cts-common', + 'zerokms-protocol', +] + +const RESEED = + 'Seed it from the root lock again: copy Cargo.lock over ' + + 'languages/python/Cargo.lock, and run `cargo metadata` in languages/python. ' + + 'If cargo moves this crate to a newer version, move it back with ' + + '`cargo update -p @ --precise `.' + +describe('the Python lock and the root lock', () => { + it('both name every crate the comparison checks', () => { + // The floor: a crate that leaves either lock would make its comparison + // below compare two empty lists, and pass. + expect( + VALUE_CRATES.filter((name) => name.startsWith('vitaminc')).length, + ).toBeGreaterThan(0) + const missing = VALUE_CRATES.filter( + (name) => !names(root).has(name) || !names(python).has(name), + ) + expect(missing, 'crates missing from one of the two locks').toEqual([]) + }) + + it.each(VALUE_CRATES)('resolve the same %s', (name) => { + const inRoot = lockedPackages(root, name) + const inPython = lockedPackages(python, name) + expect( + inPython, + `${PYTHON_LOCK} resolves ${name} ${JSON.stringify(inPython)}, and ` + + `${ROOT_LOCK} resolves ${JSON.stringify(inRoot)}. ${RESEED}`, + ).toEqual(inRoot) + }) +}) From 6ae2866922dcde5e43144edd5b54cd94d64781d6 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 18:22:56 +0900 Subject: [PATCH 10/21] ci(python): give the CS_* credentials to the live test step alone The live job set the four CS_* credentials for the whole job. So every step could read them, the builds included. A build runs third-party code: each dependency's build.rs and proc-macros, and maturin. The credentials now go to the Live tests step only. A step before it builds the extension and the EQL bundle, with no credentials. The live tests run through test:live:run, a new task that builds nothing. build:eql-bundle is the other new task. test:live still builds and runs everything, for local use. python-ci.test.mjs requires CI to run every Python task. test:live is exempt, with its reason, because CI runs it in parts. maturin is still installed unlocked (review thread 1582-F1). It now runs without the credentials. Addresses review thread 1499-F1, and in part 1582-F1. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests-python.yml | 26 ++++++++++++++++------- AGENTS.md | 2 +- languages/python/mise.toml | 31 ++++++++++++++++++++-------- scripts/__tests__/python-ci.test.mjs | 19 ++++++++++++++++- 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index 6ab8856d5..e69b3039b 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -220,11 +220,10 @@ jobs: if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' }} runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 45 - env: - CS_WORKSPACE_CRN: ${{ vars.CS_WORKSPACE_CRN }} - CS_CLIENT_ID: ${{ vars.CS_CLIENT_ID }} - CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }} - CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }} + # The CS_* credentials are given to the `Live tests` step alone, not the + # job. Every other step, the builds above all, runs third-party code: each + # dependency's build.rs and proc-macros, and maturin. None of it runs with + # the credentials set. services: postgres: image: postgres:17 @@ -288,12 +287,23 @@ jobs: with: workspaces: packages/eql - # Builds the wheel and the bundle, installs the bundle, and runs the - # live tests; a skip fails the task. + # The builds, with no credentials: the extension into .venv, and the + # EQL v4 bundle. + - name: Build the extension and the EQL bundle + run: | + mise run build + mise run build:eql-bundle + + # Installs the bundle and runs the live tests, building nothing; a skip + # fails the task. The only step with the credentials. - name: Live tests env: + CS_WORKSPACE_CRN: ${{ vars.CS_WORKSPACE_CRN }} + CS_CLIENT_ID: ${{ vars.CS_CLIENT_ID }} + CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }} + CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }} EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres sslmode=disable" - run: mise run test:live + run: mise run test:live:run # One abi3-py39 wheel per platform the FFI ships for, by the same Rust # triples. python-ci.test.mjs holds the list to the protect-ffi platform diff --git a/AGENTS.md b/AGENTS.md index 3fe4f70e6..9f6956013 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin, never in a crate manifest, so `cargo test` links libpython. `pyproject.toml` sets it, and `test:deterministic` names it again in its `--features` list; `scripts/__tests__/python-ci.test.mjs` holds the two lists together. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It also runs one test binary of `crates/core`, `deterministic_switch`, with the `deterministic-kms` feature: the switch is for the whole process, so that test needs a process of its own. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so the crates that decide what a stored value is resolve to the versions the root tests: every `vitaminc` crate, `recipher`, `ore-rs`, `clww-ore`, `cts-common` and `zerokms-protocol`. `scripts/__tests__/python-lock-parity.test.mjs` compares those crates in the two locks, by version, source and checksum. Other shared crates may differ. To re-seed, copy the root `Cargo.lock` over it and run `cargo metadata` there. That command prunes the copy to what the workspace uses. It can also move a crate to a newer version, when this workspace turns on a feature the root does not. Move each such crate back with `cargo update -p @ --precise `, until the parity test passes. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. A third job, `wheels`, builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin, never in a crate manifest, so `cargo test` links libpython. `pyproject.toml` sets it, and `test:deterministic` names it again in its `--features` list; `scripts/__tests__/python-ci.test.mjs` holds the two lists together. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It also runs one test binary of `crates/core`, `deterministic_switch`, with the `deterministic-kms` feature: the switch is for the whole process, so that test needs a process of its own. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so the crates that decide what a stored value is resolve to the versions the root tests: every `vitaminc` crate, `recipher`, `ore-rs`, `clww-ore`, `cts-common` and `zerokms-protocol`. `scripts/__tests__/python-lock-parity.test.mjs` compares those crates in the two locks, by version, source and checksum. Other shared crates may differ. To re-seed, copy the root `Cargo.lock` over it and run `cargo metadata` there. That command prunes the copy to what the workspace uses. It can also move a crate to a newer version, when this workspace turns on a feature the root does not. Move each such crate back with `cargo update -p @ --precise `, until the parity test passes. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs the live tests: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It builds the extension and the bundle in a step with no credentials, then runs `mise run test:live:run`, which builds nothing, in the only step that has them. So no dependency's build script runs with the credentials set. Locally, `mise run test:live` does both. It skips on a fork's pull request, and the task fails on any skipped test. A third job, `wheels`, builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `languages/python/examples/customer-directory`: An internal FastAPI example of the Python SDK, written twice: with psycopg (`app.py`) and with SQLAlchemy (`app_sqlalchemy.py`). It has its own `pyproject.toml` and `uv.lock`, and installs `stackencrypt` from this checkout. Its tests need real credentials, and CI does not run them. diff --git a/languages/python/mise.toml b/languages/python/mise.toml index f0876d8fa..36e50f496 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -90,20 +90,33 @@ cargo test --locked --workspace cargo test --locked -p stackencrypt-core --features deterministic-kms --test deterministic_switch """ +[tasks."build:eql-bundle"] +description = "Build the EQL v4 bundle from the tree, for the live tests" +# Built from packages/eql, as the Go task go:test:postgres builds it: the +# copy under crates/eql-bindings/sql is the released bundle and moves only at +# the lockstep bump. +run = "mise -C ../../packages/eql run build" + [tasks."test:live"] -description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names, and run the live tests against it and ZeroKMS; needs the four CS_* credentials" -depends = ["build"] +description = "Build the extension and the EQL v4 bundle, then run test:live:run; needs the four CS_* credentials and EQL_TEST_DATABASE_URL" +depends = ["build", "build:eql-bundle"] +run = "mise run test:live:run" + +[tasks."test:live:run"] +description = "Install the EQL v4 bundle into the database EQL_TEST_DATABASE_URL names, and run the live tests against it and ZeroKMS; builds nothing" shell = "bash -c" -# The bundle is built from packages/eql, as the Go task go:test:postgres -# builds it: the copy under crates/eql-bindings/sql is the released bundle -# and moves only at the lockstep bump. The tests skip without credentials or -# a database, so a skip here is a failure: this task is where both are -# given. +# Builds nothing, so CI can run the builds in a step with no credentials and +# give the credentials to this task alone: a dependency's build script then +# never runs with them set (tests-python.yml's live job). It needs the +# extension in .venv and the bundle under packages/eql/release first; locally, +# run test:live, which builds both. The tests skip without credentials or a +# database, so a skip here is a failure: this task is where both are given. run = """ set -euo pipefail : "${EQL_TEST_DATABASE_URL:?set EQL_TEST_DATABASE_URL to a libpq connection string for a disposable PostgreSQL}" -mise -C ../../packages/eql run build -psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f ../../packages/eql/release/cipherstash-encrypt-v4.sql +bundle=../../packages/eql/release/cipherstash-encrypt-v4.sql +[ -f "$bundle" ] || { echo "error: $bundle is missing; run mise run build:eql-bundle first" >&2; exit 1; } +psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f "$bundle" log="${TMPDIR:-/tmp}/stackencrypt-live.log" .venv/bin/python -m pytest -m live -v -rs | tee "$log" if grep -q ' SKIPPED' "$log"; then diff --git a/scripts/__tests__/python-ci.test.mjs b/scripts/__tests__/python-ci.test.mjs index 8e038038d..9b5df3f85 100644 --- a/scripts/__tests__/python-ci.test.mjs +++ b/scripts/__tests__/python-ci.test.mjs @@ -286,8 +286,25 @@ describe('tests-python.yml', () => { new RegExp(`(^|[\\s;&|])${command}(\\s|$)`, 'm').test(run), ) + // A local shortcut CI runs in parts. `test:live` builds, then runs + // `test:live:run`. The live job runs the builds and `test:live:run` as + // separate steps, so that no build runs with the CS_* credentials set. + const localOnly = { 'test:live': 'mise run test:live:run' } + for (const [task, runs] of Object.entries(localOnly)) { + const body = miseToml.match( + new RegExp( + `^\\[tasks\\."${task}"\\]\\n([\\s\\S]*?)(?=^\\[|(?![\\s\\S]))`, + 'm', + ), + ) + expect(body, `${task} is still a task`).not.toBeNull() + expect(body[1], `${task} still runs ${runs}`).toContain(runs) + } + const wanted = [ - ...tasks.filter((t) => !dependedOn.has(t)).map((t) => `mise run ${t}`), + ...tasks + .filter((t) => !dependedOn.has(t) && !(t in localOnly)) + .map((t) => `mise run ${t}`), 'cargo fmt --all --check', 'cargo clippy', ] From 138ddb63378f65517ea4eb680bf176b6e9472a9a Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 18:55:14 +0900 Subject: [PATCH 11/21] fix(python): let the fuzz seeds reach the value frame read check The value_frame target read its input as an arbitrary Input enum, and the committed seeds are plain value frames. arbitrary reads the first four bytes as the enum tag. So six seeds decoded as Write, and the seventh as a Read of one byte. No seed reached the read check, and CI's replay never ran it. Putting the Read tag in front of each seed is not enough. arbitrary 1.5.0 puts a byte of its own before each Vec element, so an arbitrary input never contains a frame byte for byte. The target now takes raw bytes, and the first byte chooses the branch. 0x00 is Read, and the rest of the input is the frame, byte for byte. Any other byte is Write, decoded through arbitrary as before. Each seed now starts with 0x00. On current nightly, all seven seeds reach the read check, and the replay passes. Addresses review thread 1511-F2. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .../core/fuzz/corpus/value_frame/valid-date | Bin 5 -> 6 bytes .../fuzz/corpus/value_frame/valid-decimal | Bin 17 -> 18 bytes .../fuzz/corpus/value_frame/valid-float32 | Bin 5 -> 6 bytes .../core/fuzz/corpus/value_frame/valid-int64 | Bin 9 -> 10 bytes .../core/fuzz/corpus/value_frame/valid-text | Bin 7 -> 8 bytes .../fuzz/corpus/value_frame/valid-timestamp | Bin 13 -> 14 bytes .../core/fuzz/corpus/value_frame/valid-true | Bin 1 -> 2 bytes .../core/fuzz/fuzz_targets/value_frame.rs | 104 +++++++++++------- 8 files changed, 62 insertions(+), 42 deletions(-) diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-date b/languages/python/crates/core/fuzz/corpus/value_frame/valid-date index 15c46c861ca85268eeb38bf8cfec6c99e1e70242..239847fc82d4df0d17ee85c5699575664d47b7b2 100644 GIT binary patch literal 6 NcmZP&YINgf0009_0ObGx literal 5 McmWe?bmL|K00T<^z diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-text b/languages/python/crates/core/fuzz/corpus/value_frame/valid-text index 77324ca537f8dfd918a36f90a09b45a86527657e..341f8995fc4534a44356deaee2fe84a2af038455 100644 GIT binary patch literal 8 PcmZSJVq#!mIJ^=70*L{6 literal 7 Ocmd;LVqjo6yb=Hchyi&3 diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-timestamp b/languages/python/crates/core/fuzz/corpus/value_frame/valid-timestamp index 0395bb4b90e61d03c9e020a21defbf35df028166..85a976842efb269c8f66c4590e5f5822a2c8a18d 100644 GIT binary patch literal 14 PcmZP&W`KeNS3=kU2QUHy literal 13 OcmWe;fPw>8Lf8QZF9HJq diff --git a/languages/python/crates/core/fuzz/corpus/value_frame/valid-true b/languages/python/crates/core/fuzz/corpus/value_frame/valid-true index fc2b5693e00bdbea3f74ac60459b1df43c295356..a903574af00b573ad9bdb2bccf8d93ed00c675de 100644 GIT binary patch literal 2 JcmZQz1^@sB00aO4 literal 1 IcmZPu000C41ONa4 diff --git a/languages/python/crates/core/fuzz/fuzz_targets/value_frame.rs b/languages/python/crates/core/fuzz/fuzz_targets/value_frame.rs index fd1a3f948..49ba3764f 100644 --- a/languages/python/crates/core/fuzz/fuzz_targets/value_frame.rs +++ b/languages/python/crates/core/fuzz/fuzz_targets/value_frame.rs @@ -14,17 +14,24 @@ // engine returns does. It must never panic. A scalar it accepts that the // write check also accepts must write back as a passthrough and read as // itself. +// +// The first input byte chooses the branch. 0x00 is Read, and the rest of the +// input is the frame, byte for byte, so a seed in `corpus/value_frame/` is +// 0x00 then a real frame. Any other byte is Write, and the rest decodes +// through `arbitrary`. `arbitrary` itself cannot carry a frame as it is: its +// `Vec` puts a byte of its own before each element. -use arbitrary::Arbitrary; +use arbitrary::{Arbitrary, Unstructured}; use libfuzzer_sys::fuzz_target; use stack_encrypt::dynamic::ValueFrame; use stackencrypt_core::values::{self, Kind, Scalar, Typed, Value}; +/// The first input byte that chooses Read; any other byte chooses Write. +const READ: u8 = 0x00; + +/// A Write input: a kind and a host value. #[derive(Arbitrary, Debug)] -enum Input { - Write(MirrorKind, Host), - Read(Vec), -} +struct Write(MirrorKind, Host); /// `Option`: `Passthrough` is `None`. #[derive(Arbitrary, Debug, Clone, Copy)] @@ -140,42 +147,55 @@ fn same(a: &Scalar, b: &Scalar) -> bool { } } -fuzz_target!(|input: Input| { - match input { - Input::Write(kind, host) => { - let (kind, value) = (kind.kind(), host.value()); - if values::scalar(kind, value).is_err() { - return; - } - let frame = ValueFrame::of(Typed(kind, value)).expect("an accepted value frames"); - let read = values::read(frame.risky_ref()).expect("a written frame reads"); - assert!( - same(&read, &expected(kind, value)), - "{read:?} for {kind:?} {value:?}" - ); - if let Some(plain) = independent(kind, value) { - assert_eq!(frame.risky_ref(), plain.risky_ref(), "{kind:?} {value:?}"); - } - } - Input::Read(bytes) => { - let Ok(read) = values::read(&bytes) else { - return; - }; - let Some(value) = read.value() else { - return; - }; - // `values::read` range-checks a timestamp, but not a date or a - // decimal: it reads any `i32` day count, and any scale and - // mantissa. `values::scalar` refuses a date outside the engine's - // range, a scale over 28, and a mantissa over 96 bits. The engine - // only returns a passthrough the SDK wrote, so read accepting - // more than write is not a defect. - if values::scalar(None, value).is_err() { - return; - } - let frame = ValueFrame::of(Typed(None, value)).expect("a read scalar writes back"); - let again = values::read(frame.risky_ref()).expect("a written frame reads"); - assert!(same(&again, &read), "{again:?} after {read:?}"); - } +fuzz_target!(|data: &[u8]| { + let Some((&branch, rest)) = data.split_first() else { + return; + }; + if branch == READ { + read_back(rest); + } else if let Ok(Write(kind, host)) = Write::arbitrary_take_rest(Unstructured::new(rest)) { + write_back(kind, host); } }); + +/// Write: an accepted value frames, reads back as itself narrowed to its +/// kind, and, for a plain Rust type, gives the bytes `ValueFrame::of` gives +/// that type. +fn write_back(kind: MirrorKind, host: Host) { + let (kind, value) = (kind.kind(), host.value()); + if values::scalar(kind, value).is_err() { + return; + } + let frame = ValueFrame::of(Typed(kind, value)).expect("an accepted value frames"); + let read = values::read(frame.risky_ref()).expect("a written frame reads"); + assert!( + same(&read, &expected(kind, value)), + "{read:?} for {kind:?} {value:?}" + ); + if let Some(plain) = independent(kind, value) { + assert_eq!(frame.risky_ref(), plain.risky_ref(), "{kind:?} {value:?}"); + } +} + +/// Read: `bytes` never panic `values::read`, and a scalar that both checks +/// accept writes back as a passthrough and reads as itself. +fn read_back(bytes: &[u8]) { + let Ok(read) = values::read(bytes) else { + return; + }; + let Some(value) = read.value() else { + return; + }; + // `values::read` range-checks a timestamp, but not a date or a decimal: + // it reads any `i32` day count, and any scale and mantissa. + // `values::scalar` refuses a date outside the engine's range, a scale + // over 28, and a mantissa over 96 bits. The engine only returns a + // passthrough the SDK wrote, so read accepting more than write is not a + // defect. + if values::scalar(None, value).is_err() { + return; + } + let frame = ValueFrame::of(Typed(None, value)).expect("a read scalar writes back"); + let again = values::read(frame.risky_ref()).expect("a written frame reads"); + assert!(same(&again, &read), "{again:?} after {read:?}"); +} From 11cf614fee9c396a010df2bc6a777165fe142fa9 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 19:28:28 +0900 Subject: [PATCH 12/21] docs(python): add the customer directory example's findings The example found gaps in the SDK and its documentation while it was built. This file records them, with the timings it measured and the README claims that held. Items 5, 6 and 7 were found before the SQLAlchemy layer existed, and the layer addresses each one. Each now has a status line that says how. Item 8 is new: when the engine refuses a value, decrypt_malformed and decrypt_kind_mismatch name the field but not the row. The Go SDK has the same gap. The suggested fix is in the engine, so that both SDKs get the row (review thread 1509-F7). Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .../examples/customer-directory/FINDINGS.md | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 languages/python/examples/customer-directory/FINDINGS.md diff --git a/languages/python/examples/customer-directory/FINDINGS.md b/languages/python/examples/customer-directory/FINDINGS.md new file mode 100644 index 000000000..15c43a5f5 --- /dev/null +++ b/languages/python/examples/customer-directory/FINDINGS.md @@ -0,0 +1,129 @@ +# Findings from the customer directory example + +Measured on 9 November 2026, on one macOS machine, against real ZeroKMS. The SDK was a dev-profile build. Do not use these times as release-build figures. + +## Gaps in the SDK or its documentation + +Items 5, 6, and 7 were found before the SQLAlchemy layer existed. Each now has a status line. + +### 1. The README does not warn about Pydantic's `ValidationError` + +`Customer` is a Pydantic model that contains plaintext. By default, Pydantic prints the rejected input in a `ValidationError`. FastAPI's default 422 response also repeats the input: the example's first run returned `"input":"bad"`. + +The README's list of what still shows plaintext names `model_dump`, `vars`, `pickle`, and others. It does not name either of these. + +The example sets `hide_input_in_errors=True` and replaces FastAPI's 422 response. `tests/test_app.py` covers both. + +Suggested action: add both to the README's list, with the setting that hides the input. + +### 2. The README has no recipe for a database-generated key + +The README covers a UUID key only. A required integer `id` on the model cannot work, because the model is encrypted before PostgreSQL creates the `id`. + +The example leaves `id` out of the model. It works, but the model then cannot return the `id`. + +Suggested action: add the database-generated case to "Store a UUID key". + +### 3. `decrypt` does not say whether it keeps row order + +Neither the `decrypt` docstring nor the README says that results come back in the order of the input rows. An application that needs a column outside the model must pair it with the decrypted result by position. + +The example avoids this by not returning `id`. I did not test the order. + +Suggested action: state the guarantee, and add a test for it. + +### 4. The README does not cover uniqueness on an encrypted column + +The README shows only a hash index, and PostgreSQL cannot make a hash index unique. This B-tree index works: + +```sql +CREATE UNIQUE INDEX customers_email_unique ON customers (eql_v4.eq_term(email)); +``` + +A second insert with the same email raises `UniqueViolation` with SQLSTATE 23505. `EXPLAIN` shows an index scan on this index for `WHERE email = %s::eql_v4.query_text_eq`, with `enable_seqscan` off. + +Suggested action: document the unique index, and say that equal emails produce equal terms. + +### 5. The README has no write or read path for SQLAlchemy + +The README shows how to declare a SQLAlchemy model, including relationships. It does not show an insert or a query. `app_sqlalchemy.py` needed three things that the README does not mention: + +- A custom column type. The tests fail with `sa.Text`, `sa.String`, or `sa.JSON` in its place. +- `sa.insert(Customer.__table__)` with the rows from `encrypt`, in place of `Session.add`. +- A `sa.cast` of the query term to `eql_v4.query_text_eq`. + +The `id` column works when it is marked `Ignore`. Search with these statements uses the unique index, as `EXPLAIN` shows with `enable_seqscan` off. + +Python ADR-0002 mentions a later SQLAlchemy layer. Until it exists, this is hand-written code for each application. + +Suggested action: add this recipe to the README, or say that the layer is planned. + +Status: addressed by the SQLAlchemy layer, `stackencrypt.sqlalchemy`. `app_sqlalchemy.py` now calls `configure_encryption([Customer])`. Its endpoints use `session.add` and `session.scalars`, with no SDK calls. The README section "Use it with the SQLAlchemy ORM" shows the write and the read. + +### 6. `Session.add` sends plaintext to PostgreSQL before it fails + +`Session.add` with a normal `Customer` object fails with a `DataError`, and no row is stored. But the plaintext has already reached the PostgreSQL server. The error text quotes the start of the value, even with `hide_parameters=True`: + +```text +DETAIL: Token "zed" is invalid. +CONTEXT: JSON data, line 1: zed... +``` + +The example logs only PostgreSQL's primary message, so this text is not in its log. An application that logs the exception would write part of the plaintext. A PostgreSQL server that logs errors could also record it. I did not check the server's log. + +Suggested action: warn about this in the README. The later SQLAlchemy layer could refuse this case before anything is sent. + +Status: addressed by the SQLAlchemy layer. Its write hook encrypts the values before SQLAlchemy sends the statement, so `Session.add` sends no plaintext. I did not check this example's PostgreSQL log again. + +### 7. A decrypted SQLAlchemy object is not connected to a session + +`select(Customer)` returns ciphertext dicts in `.email` and `.name`. `decrypt` returns new objects that SQLAlchemy reports as transient, with `id` set to `None`. So the usual pattern of loading an object, changing it, and committing it does not work. + +Suggested action: state this limit in the README. + +Status: addressed by the SQLAlchemy layer. A session query builds its objects as usual. The read hook then decrypts them in place with `set_committed_value`, so they are not marked as changed (ADR-0004). `tests/test_sqlalchemy_live.py` checks `session.dirty` after a load. + +### 8. Two `decrypt` errors from the engine do not name the row + +Most `decrypt` errors name the row and the field, such as `row 317, field "email"`. Two codes name the field only, when the engine finds the problem: + +- `decrypt_malformed`, for a value with an `i` that is not the field's EQL type. The engine refuses it before any key request. +- `decrypt_kind_mismatch`, for a value sealed as another type. The engine finds it after the value opens, so after the key requests. + +In a batch of 500 rows, the user must then search every row for the bad value. The Go SDK has the same gap: `Client.open` in `languages/golang/encrypt/records.go` returns the engine's error unchanged. Review thread 1509-F7 reported it for Python. + +The SDK could find the row for `decrypt_malformed` by running the engine's check again one row at a time. That needs a copy of every row's ciphertext on every call, and it cannot help with `decrypt_kind_mismatch` without new key requests. It would also make Python differ from Go. + +Suggested action: have the engine name the row in both refusals, so that Python and Go both get it. Issue #1508 already asks the engine to name the field in `decrypt_kind_mismatch`. Until then, state in the README and the `Cipher.decrypt` docstring that these two codes do not name the row. + +## Timings + +| Step | Time | +|---|---| +| `import app`, including `Cipher(Customer)` | 0.15 s | +| First `encrypt` of one customer, two runs | 0.95 s and 1.25 s | +| Next `encrypt` of one customer | about 0.22 s | +| `query` after the first call | 0.002 s | +| `psycopg.connect` and close | 0.011 s | +| 20 single-customer `encrypt` calls in sequence | 7.5 s | +| 20 single-customer `encrypt` calls on 20 threads | 0.73 s | + +The first request after start-up costs about 1 second more than later requests. The example has no warm-up call, as decided. + +## README claims that held + +- One `Cipher` can be shared between threads with no lock. The concurrency test passed, and the timings above show the calls overlap. +- `query` makes no noticeable request after the first call. It took 0.002 s. +- A Pydantic class works as both the FastAPI request body and the `stackencrypt` model. `default_factory` is accepted. +- `repr` shows `` for encrypted fields. +- Rows written by the psycopg app decrypt in the SQLAlchemy app, and the reverse. A test covers both directions. +- `decrypt` is all or nothing. One row stored under another table name made a whole list request fail with `decrypt_foreign_row`. +- Missing credentials stop the app at start-up, with `stack_auth_not_authenticated` and a message that says how to log in. + +## Limits of this example + +- It was not run on Python 3.9, the SDK's oldest supported version. It ran on 3.14 only. +- The statement that uv does not notice a Rust change is not tested. `mise run setup` always reinstalls the SDK. +- Dependabot does not update this example's `uv.lock`. +- CI does not run the tests, so the example can break with no CI failure. +- The tests print a Starlette warning that recommends `httpx2` in place of `httpx`. I left `httpx` in place, because I could not verify `httpx2`. From d7cb6ddb88a1b48209cd83192d4ca39f6794a666 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 19:31:09 +0900 Subject: [PATCH 13/21] docs(python): say which decrypt errors do not name the row Most decrypt errors name the row and the field. decrypt_malformed and decrypt_kind_mismatch from the engine name the field only, because the engine does not report the row. The README and the Cipher.decrypt docstring now say so, and say where to look for the bad value. The fix belongs in the engine, so that the Python and Go SDKs both get the row. FINDINGS.md item 8 records it. Addresses review thread 1509-F7. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 7 +++++++ languages/python/python/stackencrypt/_cipher.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/languages/python/README.md b/languages/python/README.md index f8380dbec..9421e26eb 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -87,6 +87,13 @@ A passthrough `datetime` field needs a `timestamptz` column. psycopg reads a `ti `decrypt` checks every row before it sends anything. A value stored under another table or column raises `DecryptError` with the code `decrypt_foreign_row`. A passthrough value of another type than the field declares raises `InvalidValueError`. When any row fails, `decrypt` returns no instances. +Each of these errors names the row and the field, such as `row 317, field "email"`. Two errors name the field only. The engine raises them. It does not report the row yet: + +- `decrypt_malformed`, for a value whose JSON is not the field's EQL type. Its `i` names this table and column. No key request is made. +- `decrypt_kind_mismatch`, for a value that opens but was sealed as another type. This comes after the key requests. + +For these two, look for the bad value in that field's column, across all the rows of the call. + ## A field that can be missing Declare a field that can be `None` with `Optional` inside the `Annotated`: diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index 996a55490..e6900ddf2 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -218,6 +218,11 @@ def decrypt(self, rows: Sequence[Mapping[str, object]]) -> List[T]: and no request is made. A passthrough value of another type than the field's raises ``InvalidValueError``; it is never converted. The call is all or nothing: no instance is built when any row fails. + + Each of these errors names the row. Two errors from the engine name + the field only, not the row: ``decrypt_malformed`` for a value that + is not the field's EQL type, and ``decrypt_kind_mismatch`` for a + value sealed as another type. """ return _engine.decrypt( self._plan, self._cls, _rows(rows, DecryptError, "decrypt_row_shape") From 260902becd20fd8cd20be1991ab17da29d55cbc3 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 19:50:47 +0900 Subject: [PATCH 14/21] refactor(python): read the endpoint and client key with the engine's own code The Python SDK builds its engine client over a connection that counts its requests, so it cannot use StackCipherBuilder. It copied two private pieces of that builder: the reader of CS_ZEROKMS_HOST in stack-kms, and the client key provider in stack-encrypt. A change to either would not reach Python, and no test would report it. stack-kms now exports endpoint_from_env, the reader the builder uses. stack-encrypt now exports client_key_provider, the provider init uses: the environment first, then the stash profile. The Python client calls both, and its copies are gone. The two copies had already drifted: with no client key and no profile, Python now gives the engine's message, which says npx stash auth login. The wasm32 variant of client_key_provider named KeyProvider, which is imported only for other targets. It now uses the full path. Nothing in CI builds stack-encrypt's http feature for wasm32. Addresses review thread 1499-F15. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/crates/core/src/client.rs | 54 ++++------------------ packages/stack-encrypt/src/cipher.rs | 11 +++-- packages/stack-encrypt/src/lib.rs | 2 + packages/stack-kms/src/builder.rs | 44 +++++++++++------- packages/stack-kms/src/lib.rs | 2 +- 5 files changed, 48 insertions(+), 65 deletions(-) diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index 9958eeadf..b7c14a09f 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -22,11 +22,10 @@ use std::sync::{Mutex, MutexGuard, OnceLock}; use stack_auth::AutoStrategy; use stack_encrypt::kms::{ - ClientOpts, ConnectionInitError, EnvKeyProvider, FallbackKeyProvider, HttpConnection, - HttpConnectionOpts, KeyProvider, KeyProviderError, ProfileStore, StackKms, - StackKmsBuilderError, ZeroKMSConnection, ZeroKMSConnectionInit, ZeroKmsEndpoint, + endpoint_from_env, ClientOpts, ConnectionInitError, HttpConnection, HttpConnectionOpts, + KeyProvider, StackKms, ZeroKMSConnection, ZeroKMSConnectionInit, ZeroKmsEndpoint, }; -use stack_encrypt::StackCipher; +use stack_encrypt::{client_key_provider, StackCipher}; use zerokms_protocol::{ViturRequest, ViturRequestError}; use crate::credentials; @@ -280,15 +279,14 @@ fn build() -> Result { async fn init() -> Result, Refusal> { // First: it reads one variable and sends nothing, so a bad value is // refused before the credentials are read. - let endpoint = endpoint_from_env()?; + // The endpoint and the client key come from the engine's own readers, + // the ones StackCipherBuilder uses, so the two cannot drift apart. + let endpoint = endpoint_from_env().map_err(|error| Refusal::of(&error))?; let credentials = AutoStrategy::detect().map_err(|error| credentials::refusal(&error))?; - let client_key = FallbackKeyProvider::new( - EnvKeyProvider, - ProfileClientKey(ProfileStore::resolve(None).ok()), - ) - .client_key() - .await - .map_err(|error| Refusal::of(&error))?; + let client_key = client_key_provider() + .client_key() + .await + .map_err(|error| Refusal::of(&error))?; // Requests go out one at a time, as the spec states the request count: // ceil(values / 500) key requests "in sequence". The Go guest sets the // same. @@ -304,38 +302,6 @@ async fn init() -> Result, Refusal> { .map_err(|error| Refusal::of(&error)) } -/// The ZeroKMS endpoint override, as `StackKmsBuilder` reads it: the first -/// of `CS_ZEROKMS_HOST` and the legacy `CS_VITUR_HOST` that is set. `None` -/// leaves the endpoint to the token's services claim, as every other SDK -/// does. `init` builds the client itself to count its requests, so it reads -/// the variable here; the builder's own reader is private. -fn endpoint_from_env() -> Result, Refusal> { - for env_var in stack_encrypt::kms::vars::CS_ZEROKMS_HOST { - if let Ok(value) = std::env::var(env_var) { - return value.parse().map(Some).map_err(|source| { - Refusal::of(&StackKmsBuilderError::InvalidEndpoint { env_var, source }) - }); - } - } - Ok(None) -} - -/// The CLI's profile as a key provider, where it resolves. Where it does -/// not, the error says what to do, as stack-encrypt's own provider does. -struct ProfileClientKey(Option); - -impl KeyProvider for ProfileClientKey { - async fn client_key(&self) -> Result { - match &self.0 { - Some(store) => store.client_key().await, - None => Err(KeyProviderError::NotConfigured( - "no client key: set CS_CLIENT_ID and CS_CLIENT_KEY, or log in with `stash auth login`" - .into(), - )), - } - } -} - /// The HTTP connection, counting every request it sends. pub struct Counting(HttpConnection); diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index c1962a854..82647d3cc 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -811,17 +811,22 @@ pub struct FromEnv; /// workspace's `secretkey.json` in the profile directory. A profile directory /// that cannot be resolved is not an error here — env-only setups (CI) have /// none — it just leaves the environment as the only source. +/// +/// This is the provider [`StackCipherBuilder::init`] uses. It is public so +/// that a client built over its own connection, as the Python SDK builds one +/// to count its requests, finds the client key the same way. #[cfg(all(feature = "http", not(target_arch = "wasm32")))] -fn client_key_provider() -> FallbackKeyProvider { +pub fn client_key_provider() -> impl KeyProvider { FallbackKeyProvider::new( EnvKeyProvider, ProfileClientKey(ProfileStore::resolve(None).ok()), ) } -/// wasm32 has no filesystem, so no profile: the environment is the only source. +/// The client key from the environment: wasm32 has no filesystem, so no +/// profile, and the environment is the only source. #[cfg(all(feature = "http", target_arch = "wasm32"))] -fn client_key_provider() -> EnvKeyProvider { +pub fn client_key_provider() -> impl stack_kms::KeyProvider { EnvKeyProvider } diff --git a/packages/stack-encrypt/src/lib.rs b/packages/stack-encrypt/src/lib.rs index d45570e82..34fff1659 100644 --- a/packages/stack-encrypt/src/lib.rs +++ b/packages/stack-encrypt/src/lib.rs @@ -282,6 +282,8 @@ pub mod sem; pub mod target; pub mod value; +#[cfg(feature = "http")] +pub use cipher::client_key_provider; pub use cipher::{ Error, FromEnv, LeafBytesError, PendingStackCipherText, SealedValue, StackCipher, StackCipherBuilder, StackCipherText, diff --git a/packages/stack-kms/src/builder.rs b/packages/stack-kms/src/builder.rs index 7d1bca258..7a277a724 100644 --- a/packages/stack-kms/src/builder.rs +++ b/packages/stack-kms/src/builder.rs @@ -227,7 +227,7 @@ impl StackKmsBuilder { fn build_opts(self) -> Result<(ClientOpts, C, S), StackKmsBuilderError> { let mut connection = self.connection; if connection.base_url().is_none() { - if let Some(endpoint) = Self::base_url_from_env()? { + if let Some(endpoint) = endpoint_from_env()? { connection = connection.with_base_url(endpoint); } } @@ -241,24 +241,34 @@ impl StackKmsBuilder { Ok((opts, self.credentials, self.client_key)) } +} - /// Resolve the ZeroKMS endpoint from the `CS_ZEROKMS_HOST` environment - /// variable (or legacy `CS_VITUR_HOST`). The first variable that is set - /// decides: an unusable value is an error, not a fall-through. - fn base_url_from_env() -> Result, StackKmsBuilderError> { - use crate::vars::CS_ZEROKMS_HOST; - - for env_var in CS_ZEROKMS_HOST { - if let Ok(value) = std::env::var(env_var) { - return value - .parse() - .map(Some) - .map_err(|source| StackKmsBuilderError::InvalidEndpoint { env_var, source }); - } +/// The ZeroKMS endpoint from the environment, as [`StackKmsBuilder`] reads +/// it: `CS_ZEROKMS_HOST`, or the legacy `CS_VITUR_HOST`. The first variable +/// that is set decides: an unusable value is an error, not a fall-through. +/// `None` when neither is set, which leaves the endpoint to the access +/// token's services claim. +/// +/// Public so that a client built over its own connection, as the Python +/// SDK builds one to count its requests, reads the endpoint the same way. +/// +/// # Errors +/// +/// [`StackKmsBuilderError::InvalidEndpoint`], naming the variable, when its +/// value is not a usable endpoint. +pub fn endpoint_from_env() -> Result, StackKmsBuilderError> { + use crate::vars::CS_ZEROKMS_HOST; + + for env_var in CS_ZEROKMS_HOST { + if let Ok(value) = std::env::var(env_var) { + return value + .parse() + .map(Some) + .map_err(|source| StackKmsBuilderError::InvalidEndpoint { env_var, source }); } - - Ok(None) } + + Ok(None) } impl StackKmsBuilder @@ -370,7 +380,7 @@ mod tests { const LEGACY: &str = "CS_VITUR_HOST"; fn from_env() -> Result, StackKmsBuilderError> { - StackKmsBuilder::::base_url_from_env() + endpoint_from_env() } #[test] diff --git a/packages/stack-kms/src/lib.rs b/packages/stack-kms/src/lib.rs index ee484f39e..30a28fdfb 100644 --- a/packages/stack-kms/src/lib.rs +++ b/packages/stack-kms/src/lib.rs @@ -95,7 +95,7 @@ pub mod vars; // Builder (configures the default HTTP transport) #[cfg(feature = "http")] -pub use builder::{StackKmsBuilder, StackKmsBuilderError, WithKeyProvider}; +pub use builder::{endpoint_from_env, StackKmsBuilder, StackKmsBuilderError, WithKeyProvider}; // Clients pub use client::{ From 75c7810558fbc11509c5ad271f3b61b23e5c35c6 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 19:57:30 +0900 Subject: [PATCH 15/21] build(python): lock maturin, and build with the locked copy uv.lock fixed the test tools, but not maturin, the build backend. Each build downloaded maturin from PyPI at any version in [build-system]'s range, with no hash check and no Dependabot cooldown. maturin is now in a build dependency group, so uv.lock records its version and hash. Dependabot updates it with the test extra. mise run build installs the group into .venv, and builds the package with --no-build-isolation, so uv downloads no maturin of its own. Without maturin in .venv, that build fails. [build-system] keeps its range, for an install by any other route. The wheels job still uses maturin-action, with maturin's version fixed in the workflow. Addresses review thread 1582-F1. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- languages/python/mise.toml | 20 ++++++++++---------- languages/python/pyproject.toml | 8 ++++++++ languages/python/uv.lock | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9f6956013..f25cbc044 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin, never in a crate manifest, so `cargo test` links libpython. `pyproject.toml` sets it, and `test:deterministic` names it again in its `--features` list; `scripts/__tests__/python-ci.test.mjs` holds the two lists together. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It also runs one test binary of `crates/core`, `deterministic_switch`, with the `deterministic-kms` feature: the switch is for the whole process, so that test needs a process of its own. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so the crates that decide what a stored value is resolve to the versions the root tests: every `vitaminc` crate, `recipher`, `ore-rs`, `clww-ore`, `cts-common` and `zerokms-protocol`. `scripts/__tests__/python-lock-parity.test.mjs` compares those crates in the two locks, by version, source and checksum. Other shared crates may differ. To re-seed, copy the root `Cargo.lock` over it and run `cargo metadata` there. That command prunes the copy to what the workspace uses. It can also move a crate to a newer version, when this workspace turns on a feature the root does not. Move each such crate back with `cargo update -p @ --precise `, until the parity test passes. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs the live tests: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It builds the extension and the bundle in a step with no credentials, then runs `mise run test:live:run`, which builds nothing, in the only step that has them. So no dependency's build script runs with the credentials set. Locally, `mise run test:live` does both. It skips on a fork's pull request, and the task fails on any skipped test. A third job, `wheels`, builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin, never in a crate manifest, so `cargo test` links libpython. `pyproject.toml` sets it, and `test:deterministic` names it again in its `--features` list; `scripts/__tests__/python-ci.test.mjs` holds the two lists together. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It also runs one test binary of `crates/core`, `deterministic_switch`, with the `deterministic-kms` feature: the switch is for the whole process, so that test needs a process of its own. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so the crates that decide what a stored value is resolve to the versions the root tests: every `vitaminc` crate, `recipher`, `ore-rs`, `clww-ore`, `cts-common` and `zerokms-protocol`. `scripts/__tests__/python-lock-parity.test.mjs` compares those crates in the two locks, by version, source and checksum. Other shared crates may differ. To re-seed, copy the root `Cargo.lock` over it and run `cargo metadata` there. That command prunes the copy to what the workspace uses. It can also move a crate to a newer version, when this workspace turns on a feature the root does not. Move each such crate back with `cargo update -p @ --precise `, until the parity test passes. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs the live tests: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It builds the extension and the bundle in a step with no credentials, then runs `mise run test:live:run`, which builds nothing, in the only step that has them. So no dependency's build script runs with the credentials set. Locally, `mise run test:live` does both. It skips on a fork's pull request, and the task fails on any skipped test. A third job, `wheels`, builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. It also locks maturin, the build backend, in the `build` dependency group. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. It then builds the package with `--no-build-isolation`, through the locked maturin in `.venv`, so uv downloads no build tool outside the lock. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. The `wheels` job builds with `PyO3/maturin-action` instead, with maturin's version fixed in the workflow. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `languages/python/examples/customer-directory`: An internal FastAPI example of the Python SDK, written twice: with psycopg (`app.py`) and with SQLAlchemy (`app_sqlalchemy.py`). It has its own `pyproject.toml` and `uv.lock`, and installs `stackencrypt` from this checkout. Its tests need real credentials, and CI does not run them. diff --git a/languages/python/mise.toml b/languages/python/mise.toml index 36e50f496..97ea93efe 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -26,9 +26,8 @@ set -euo pipefail # recreated, so a change to `[tools]` cannot leave the tests on the old one. # The `test` extra comes from uv.lock, with --locked, so a lock that does not # match pyproject.toml fails the build. Dependabot updates the lock. The -# package has no runtime dependency, so the lock fixes test tooling versions -# alone. maturin, the build backend, is not in the lock. uv resolves it from -# the index in an isolated build environment, within pyproject.toml's range. +# package has no runtime dependency, so the lock fixes tool versions alone: +# the test extra, and maturin, the build backend, in the `build` group. want="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')" have="$([ -x .venv/bin/python ] && .venv/bin/python -c 'import sys; print("%d.%d" % sys.version_info[:2])' || true)" if [ "$have" != "$want" ]; then @@ -40,13 +39,14 @@ fi # `wheels` job in tests-python.yml builds release, because the time it records # is the cost of the wheel a user would install. export MATURIN_PEP517_ARGS="${MATURIN_PEP517_ARGS:---profile dev}" -# The `test` extra at the locked versions. The project itself is left out: -# uv sync would install it editable. -uv sync --locked --extra test --no-install-project --python .venv/bin/python -# The package itself, built by maturin through the PEP 517 backend. Always -# reinstall it so a Rust change is picked up. Its dependencies are the ones -# above, so none is resolved here. -uv pip install --python .venv/bin/python --reinstall-package stackencrypt --no-deps . +# The `test` extra and the `build` group at the locked versions. The project +# itself is left out: uv sync would install it editable. +uv sync --locked --extra test --group build --no-install-project --python .venv/bin/python +# The package itself, built by the locked maturin in .venv through the PEP +# 517 backend. --no-build-isolation keeps uv from downloading a maturin of +# its own, outside the lock. Always reinstall it so a Rust change is picked +# up. Its dependencies are the ones above, so none is resolved here. +uv pip install --python .venv/bin/python --no-build-isolation --reinstall-package stackencrypt --no-deps . .venv/bin/python -c 'import stackencrypt as s; print("stackencrypt", s.__version__, "engine", s.__engine__)' """ diff --git a/languages/python/pyproject.toml b/languages/python/pyproject.toml index 191850a11..00a11c56c 100644 --- a/languages/python/pyproject.toml +++ b/languages/python/pyproject.toml @@ -35,6 +35,14 @@ test = [ "alembic>=1.13", ] +# maturin, the build backend, for `mise run build`. uv.lock records its +# version and hash, and Dependabot updates it with the test extra, after the +# same 7-day cooldown. The build installs it into .venv and builds with +# --no-build-isolation, so uv downloads no unlocked build tool. The range is +# [build-system]'s, which stays for an install by any other route. +[dependency-groups] +build = ["maturin>=1.9,<2.0"] + [tool.maturin] # The pure-Python package is the distribution; the Rust extension is its # `_engine` submodule, so the Python side can change without a recompile. diff --git a/languages/python/uv.lock b/languages/python/uv.lock index e00021c0e..f41a44589 100644 --- a/languages/python/uv.lock +++ b/languages/python/uv.lock @@ -460,6 +460,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/7d/87ff065a4d14d8de1f462f851d9cdf903b6e1fc067d0ef3674414da444c0/markupsafe-3.0.4-cp39-cp39-win_arm64.whl", hash = "sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378", size = 14190, upload-time = "2026-10-02T23:07:20.567Z" }, ] +[[package]] +name = "maturin" +version = "1.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/71/eb078174335ff30bde1be2ad54fb6256db4b160eb3efa81163fa30f8c814/maturin-1.16.0.tar.gz", hash = "sha256:13e69ca12542999d8c5467131de72a49bb76e4c284b9a45c5783df7c0fcb3a8a", size = 396641, upload-time = "2026-10-29T14:09:43.096Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/f8/235d4a8b3d61af3c7744582ae9e6ccb8f096e9f68f06fe62d7d03e5d7b2d/maturin-1.16.0-py3-none-linux_armv6l.whl", hash = "sha256:deacbb204a20326862e194c7df4958e0514a0411c9a3ccf228a00067c11b5c7e", size = 10425070, upload-time = "2026-10-29T14:09:06.221Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d5/f49a90d8c489dc5f134654ec91d12a78a51599fdc8c2b5a1bc2391bca0ac/maturin-1.16.0-py3-none-macosx_10_12_universal2.macosx_10_12_x86_64.macosx_11_0_arm64.whl", hash = "sha256:131fcc17ddb3ccc48aa183d2172ebf994c322b858e1b9694fb8744d783df66f6", size = 19839189, upload-time = "2026-10-29T14:09:09.441Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8e/d65633d7c25484cb82735a51e15bc969f9874a13557e2b352d27df574c20/maturin-1.16.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2216d731c9324869d3bff72c0c73009bb6e640b3fa7d340174ae70dc874d1d9f", size = 10271659, upload-time = "2026-10-29T14:09:12.596Z" }, + { url = "https://files.pythonhosted.org/packages/0a/dc/e77f490557cb5e045a75c3d18a7965425ac576235c970dfff4d903eb2c74/maturin-1.16.0-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:837db2f4d93d4899c3ac393fb3cc9eebf35b6002336e5d574f4a4fb6c49fe283", size = 10763361, upload-time = "2026-10-29T14:09:16.11Z" }, + { url = "https://files.pythonhosted.org/packages/fa/eb/535564a4211105fd64611fa4f9437c8925eded26a97d58de84565f5aebfa/maturin-1.16.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:399fadcd4af39532a173e010240c34930eacc1bb1746bb179863481c68bcf1ab", size = 10442856, upload-time = "2026-10-29T14:09:18.64Z" }, + { url = "https://files.pythonhosted.org/packages/45/5b/570e13ce81f103be6609c067a06c5bac08da5af0cf7ea700c9ae43419be9/maturin-1.16.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:e285ec9eb10243d45d10417178cb2eb76e0cbb184d0aa4378b503489fa12ee78", size = 10298796, upload-time = "2026-10-29T14:09:21.385Z" }, + { url = "https://files.pythonhosted.org/packages/88/a6/26b40c31c354359fe4e49398113d018694c7035968fe1b569bb13c3905af/maturin-1.16.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:609228d639c5b888dc41364cbd318547a59f027481f53cba85838f46da4b3f02", size = 10268023, upload-time = "2026-10-29T14:09:23.94Z" }, + { url = "https://files.pythonhosted.org/packages/2b/19/faca1784d917c2eecd498648dff6a0179b299c8142cc448ea5b1183545ec/maturin-1.16.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:7001fd3ac591820d01f2fa041552c367ec587c98649d9f10a817cc7f28e59062", size = 13739038, upload-time = "2026-10-29T14:09:26.662Z" }, + { url = "https://files.pythonhosted.org/packages/36/c1/d67e77aa07100e3b2d4c0d9ef6922646d9c4bab25cc04f91023aefc72418/maturin-1.16.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b0573a146803f24566f5b301a4572a8e49bcab4917175407c730da13b46e134", size = 11100193, upload-time = "2026-10-29T14:09:29.659Z" }, + { url = "https://files.pythonhosted.org/packages/76/44/ccef52f02ab0e1f7258bc8fe0140b4991c6287458ef883af1027f946a253/maturin-1.16.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:c16d3589cbddda9420237fdb1394b1a9b36483c0bce83b23de6118c908a8e940", size = 10663849, upload-time = "2026-10-29T14:09:32.183Z" }, + { url = "https://files.pythonhosted.org/packages/6e/9b/3104f62dbb428f9e26e35cba2c80cfc42ba9bb1dc4968882c19d1bfe594b/maturin-1.16.0-py3-none-win32.whl", hash = "sha256:9e221c047931caa79283cc0c2278904ae1b25880dde3574b0447657d72907689", size = 9124894, upload-time = "2026-10-29T14:09:34.857Z" }, + { url = "https://files.pythonhosted.org/packages/63/b8/c25ce706c4797ee1663bcf1f386ffe91d8a3b3589700a135a204b49bc4b0/maturin-1.16.0-py3-none-win_amd64.whl", hash = "sha256:9d43a2adf51a2794e907a8f7dbad06eb114d5edb0909c73c1fa9040f15e584e9", size = 10575645, upload-time = "2026-10-29T14:09:37.864Z" }, + { url = "https://files.pythonhosted.org/packages/eb/74/f69aba8b22539e5ed58323d4a6ee974afd9edda238489c73926cc0fc9d51/maturin-1.16.0-py3-none-win_arm64.whl", hash = "sha256:95f34ce53249fb7095ff4f2aff64d2562c511d6edf3e216c15dbffcd82b72b4f", size = 9886475, upload-time = "2026-10-29T14:09:40.603Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -1225,6 +1249,11 @@ test = [ { name = "sqlalchemy", version = "2.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] +[package.dev-dependencies] +build = [ + { name = "maturin" }, +] + [package.metadata] requires-dist = [ { name = "alembic", marker = "extra == 'test'", specifier = ">=1.13" }, @@ -1236,6 +1265,9 @@ requires-dist = [ ] provides-extras = ["test"] +[package.metadata.requires-dev] +build = [{ name = "maturin", specifier = ">=1.9,<2.0" }] + [[package]] name = "tomli" version = "2.5.0" From 8187e43769574152f26e913fa8ef7845b585ad20 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 20:32:45 +0900 Subject: [PATCH 16/21] chore(ci): say that Dependabot updates the locked maturin The comment on the uv entry said uv.lock contains only the test extra. It now also locks maturin in the build dependency group. Dependabot's uv parser reads [dependency-groups] (uv/helpers/lib/parser.py in dependabot-core), so maturin gets the same updates and cooldown. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f77930088..27addfa07 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -369,12 +369,15 @@ updates: update-types: - version-update:semver-major - # ── uv (languages/python — the Python SDK's test dependencies) ── - # The package has no runtime dependency, so uv.lock holds only the `test` - # extra: pytest, attrs, Pydantic, SQLAlchemy and psycopg, and what they - # need. tests-python.yml installs it with `uv sync --locked`. The extra has - # lower bounds and no upper bounds, so a new minor or patch release changes - # only the lock. + # ── uv (languages/python — the Python SDK's test and build tools) ── + # The package has no runtime dependency. So uv.lock contains two sets of + # tools: the `test` extra (pytest, attrs, Pydantic, SQLAlchemy, psycopg and + # Alembic), and the `build` dependency group (maturin, the build backend), + # with what they need. `mise run build` installs both with + # `uv sync --locked`. Dependabot's uv parser reads `[dependency-groups]` + # as well as the extras, so maturin is updated with the rest. Both have + # lower bounds and no major upper bound, so a new minor or patch release + # changes only the lock. - package-ecosystem: uv directory: /languages/python # Monthly, matching the other non-npm toolchains. No `day:`, for the From 5b9b1fdcfbb98d3a9716c4a3fb1c18637a24795c Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 20:33:13 +0900 Subject: [PATCH 17/21] fix(python): clearer refusal messages, and seal Pydantic's other print hooks - Pydantic builds __repr_str__ and devtools' __pretty__ from __repr_args__, which still showed plaintext. It is now replaced on a Pydantic model with the sealed fields. - Refusal::of joined an engine message and its help with a space, so the two read as one broken sentence. A full stop now ends the message. - NotAuthenticated is also raised for a profile that lacks its region or client ID. Its message no longer says that nothing was found. - Core's query_unsupported now names the marker in reason, as the Python check does. Addresses review threads 1453-F3, F4, F5, and 1510-F1. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .../python/crates/core/src/credentials.rs | 7 ++- languages/python/crates/core/src/query.rs | 26 +++++++++- languages/python/crates/core/src/refusal.rs | 49 +++++++++++++++++-- .../python/python/stackencrypt/_redact.py | 10 ++++ languages/python/tests/test_redaction.py | 8 +++ 5 files changed, 93 insertions(+), 7 deletions(-) diff --git a/languages/python/crates/core/src/credentials.rs b/languages/python/crates/core/src/credentials.rs index a0d496a4c..5bae0296d 100644 --- a/languages/python/crates/core/src/credentials.rs +++ b/languages/python/crates/core/src/credentials.rs @@ -49,8 +49,11 @@ pub fn refusal(error: &AuthError) -> Refusal { /// no variable at all. `None` keeps the engine's message and help. fn message(error: &AuthError) -> Option { let text = match error { - AuthError::NotAuthenticated(_) => "no credentials were found. Set CS_CLIENT_ACCESS_KEY \ - and CS_WORKSPACE_CRN, or log in with `stash auth login`." + // Also raised for a profile that was found but lacks its region or + // client ID, so the message does not say that nothing was found. + AuthError::NotAuthenticated(_) => "no usable credentials were found: none are set, or \ + the profile from `stash auth login` is incomplete. Set CS_CLIENT_ACCESS_KEY and \ + CS_WORKSPACE_CRN, or log in again with `stash auth login`." .to_owned(), AuthError::MissingWorkspaceCrn(_) => { "CS_CLIENT_ACCESS_KEY is set and CS_WORKSPACE_CRN is not. Set both.".to_owned() diff --git a/languages/python/crates/core/src/query.rs b/languages/python/crates/core/src/query.rs index dee96490b..5ea210df0 100644 --- a/languages/python/crates/core/src/query.rs +++ b/languages/python/crates/core/src/query.rs @@ -177,11 +177,23 @@ pub fn answers(plan: &ModelPlan, field: &str, op: OpSpec) -> Result<(), Refusal> if operators(plan, field)?.contains(&op.name()) { return Ok(()); } + // The marker, as the Python check names it: the EQL type, or + // Passthrough for a field with none. + let marker = plan + .fields() + .iter() + .find(|declared| declared.name == field) + .map(|declared| { + declared + .eql + .as_ref() + .map_or_else(|| "Passthrough".to_owned(), |(name, _)| name.clone()) + }); Err(Refusal { code: "query_unsupported".to_owned(), message: format!("field {field:?} has no index that answers {:?}.", op.name()), field: Some(field.to_owned()), - reason: None, + reason: marker, }) } @@ -194,6 +206,18 @@ mod tests { use crate::decrypt::tests::{cipher, run, stored, users}; use crate::encrypt::{Column, Field}; + /// The refusal names the field's marker in `reason`, as the Python + /// check does: the EQL type, or `Passthrough`. + #[test] + fn query_unsupported_names_the_marker_in_reason() { + let refused = answers(&users(), "email", OpSpec::Lt).unwrap_err(); + assert_eq!(refused.code, "query_unsupported"); + assert_eq!(refused.reason.as_deref(), Some("TextEq")); + + let refused = answers(&users(), "age", OpSpec::Eq).unwrap_err(); + assert_eq!(refused.reason.as_deref(), Some("Passthrough")); + } + fn term(cipher: &StackCipher, email: &str) -> serde_json::Value { let json = run(query_with( &cipher.default_keyset(), diff --git a/languages/python/crates/core/src/refusal.rs b/languages/python/crates/core/src/refusal.rs index ed7b31fea..989fd4a40 100644 --- a/languages/python/crates/core/src/refusal.rs +++ b/languages/python/crates/core/src/refusal.rs @@ -18,11 +18,15 @@ pub struct Refusal { } impl Refusal { - /// The code and message of `error`, with no field or reason. + /// The code and message of `error`, with no field or reason. The + /// engine's help follows the message as a sentence of its own: most + /// engine messages end with no full stop. pub fn of(error: &dyn Diagnostic) -> Self { + let text = error.to_string(); let message = match error.help() { - Some(help) => format!("{error} {help}"), - None => error.to_string(), + Some(help) if text.ends_with(['.', '!', '?']) => format!("{text} {help}"), + Some(help) => format!("{text}. {help}"), + None => text, }; Self { code: error.code().map_or_else( @@ -44,7 +48,44 @@ pub fn python_code(engine_code: &str) -> String { #[cfg(test)] mod tests { - use super::python_code; + use super::{python_code, Refusal}; + + /// An engine error with a message and a help, written by hand: this + /// crate has no derive for errors. + #[derive(Debug)] + struct WithHelp { + message: &'static str, + } + + impl std::fmt::Display for WithHelp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.message) + } + } + + impl std::error::Error for WithHelp {} + + impl miette::Diagnostic for WithHelp { + fn code<'a>(&'a self) -> Option> { + Some(Box::new("stack_test::joined")) + } + + fn help<'a>(&'a self) -> Option> { + Some(Box::new("Do this.")) + } + } + + #[test] + fn the_message_and_its_help_are_two_sentences() { + let joined = Refusal::of(&WithHelp { + message: "the plan has no field", + }); + assert_eq!(joined.message, "the plan has no field. Do this."); + let ended = Refusal::of(&WithHelp { + message: "the plan has no field.", + }); + assert_eq!(ended.message, "the plan has no field. Do this."); + } #[test] fn engine_codes_replace_the_path_separator() { diff --git a/languages/python/python/stackencrypt/_redact.py b/languages/python/python/stackencrypt/_redact.py index e733d9dbe..39d1eff96 100644 --- a/languages/python/python/stackencrypt/_redact.py +++ b/languages/python/python/stackencrypt/_redact.py @@ -120,6 +120,16 @@ def __rich_repr__(self: object) -> Iterator[Tuple[str, object]]: for method in methods: method.__qualname__ = f"{cls.__qualname__}.{method.__name__}" setattr(cls, method.__name__, method) + # Pydantic builds its other print hooks from __repr_args__: __repr_str__, + # and __pretty__, which devtools' debug() prints through. It is replaced + # too, with the same fields. Cipher does not check it again. + if isinstance(getattr(cls, "__pydantic_fields__", None), dict): + + def __repr_args__(self: object) -> Iterator[Tuple[str, object]]: + return __rich_repr__(self) + + __repr_args__.__qualname__ = f"{cls.__qualname__}.__repr_args__" + setattr(cls, "__repr_args__", __repr_args__) return methods diff --git a/languages/python/tests/test_redaction.py b/languages/python/tests/test_redaction.py index f4db8be87..879c7386e 100644 --- a/languages/python/tests/test_redaction.py +++ b/languages/python/tests/test_redaction.py @@ -125,6 +125,14 @@ def test_repr_and_str_seal_every_encrypted_field(declare): assert list(user.__rich_repr__()) == [("email", SEALED), ("age", 42)] +def test_every_pydantic_print_hook_seals_the_encrypted_fields(): + # Pydantic builds __repr_str__, __pretty__ (devtools' debug()) and + # __rich_repr__ from __repr_args__. + user = declare_pydantic() + assert list(user.__repr_args__()) == [("email", SEALED), ("age", 42)] + assert SECRET not in user.__repr_str__(", ") + + def test_a_hand_written_repr_is_refused(): def build(): @stackencrypt.model("users") From af68a555e7f100490ae438656d8303f82c4c4621 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 20:33:42 +0900 Subject: [PATCH 18/21] test(python): pin the value numbers, the commit rules, and the generator guards - A date and a datetime read as known engine numbers, before the epoch and with a non-UTC offset, and back. A decimal reads as its exact mantissa and scale, signs and positive exponents included. - The STACKENCRYPT_ENGINE_COMMIT rules moved from build.rs into engine_commit_rules.rs, which build.rs and a unit test both include. - The frame reader reads a batch written the way dynamic::decrypt writes one. - The _eql.py generator's guards have tests. A new py_ident check refuses a catalog name that is not a usable Python class name. Addresses review threads 1405-F3, 1414-F5, 1415-F2, F3, and 1499-F13, F14. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/crates/core/build.rs | 15 ++- .../python/crates/core/engine_commit_rules.rs | 16 ++++ languages/python/crates/core/src/lib.rs | 25 +++++ .../crates/core/tests/gate_frame_reader.rs | 51 +++++++++- languages/python/crates/engine/src/values.rs | 93 +++++++++++++++++++ packages/eql/crates/eql-codegen/src/py_eql.rs | 69 +++++++++++++- 6 files changed, 259 insertions(+), 10 deletions(-) create mode 100644 languages/python/crates/core/engine_commit_rules.rs diff --git a/languages/python/crates/core/build.rs b/languages/python/crates/core/build.rs index 5aa0aa46c..21908a4f8 100644 --- a/languages/python/crates/core/build.rs +++ b/languages/python/crates/core/build.rs @@ -21,6 +21,9 @@ //! checkout there is nothing to check it against. With neither, the build //! fails: an "unknown" engine in a wheel is a report nobody can act on. +#[path = "engine_commit_rules.rs"] +mod engine_commit_rules; + use std::env; use std::path::{Path, PathBuf}; use std::process::Command; @@ -36,13 +39,15 @@ const DEPENDENCY_FILES: [&str; 2] = ["../../../../Cargo.toml", "../../Cargo.lock fn main() { println!("cargo:rerun-if-env-changed={ENV_OVERRIDE}"); - let (commit, source) = match env::var(ENV_OVERRIDE) { - Ok(value) if !value.is_empty() => (value.to_ascii_lowercase(), "env"), - _ => (commit_from_git(), "git"), + let given = env::var(ENV_OVERRIDE) + .ok() + .and_then(|value| engine_commit_rules::from_override(&value)); + let (commit, source) = match given { + Some(commit) => (commit, "env"), + None => (commit_from_git(), "git"), }; - let sha = commit.strip_suffix("-dirty").unwrap_or(&commit); assert!( - sha.len() == 40 && sha.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')), + engine_commit_rules::is_valid(&commit), "the engine commit must be a full 40-character hex SHA (optionally `-dirty`), got {commit:?}" ); println!("cargo:rustc-env={ENV_OVERRIDE}={commit}"); diff --git a/languages/python/crates/core/engine_commit_rules.rs b/languages/python/crates/core/engine_commit_rules.rs new file mode 100644 index 000000000..59784a67f --- /dev/null +++ b/languages/python/crates/core/engine_commit_rules.rs @@ -0,0 +1,16 @@ +//! The rules for the engine commit that `build.rs` records. `build.rs` +//! includes this file with `mod`, and the crate's tests include it with +//! `#[path]`, so the rules are tested without running a build. + +/// The commit given in `STACKENCRYPT_ENGINE_COMMIT`, in lower case, or +/// `None` when the value is empty: an empty value means "read git". +pub fn from_override(value: &str) -> Option { + (!value.is_empty()).then(|| value.to_ascii_lowercase()) +} + +/// Whether `commit` is a full 40-character lower-case hex SHA, with +/// `-dirty` after it or not. +pub fn is_valid(commit: &str) -> bool { + let sha = commit.strip_suffix("-dirty").unwrap_or(commit); + sha.len() == 40 && sha.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} diff --git a/languages/python/crates/core/src/lib.rs b/languages/python/crates/core/src/lib.rs index c8273e59f..a404dd6f9 100644 --- a/languages/python/crates/core/src/lib.rs +++ b/languages/python/crates/core/src/lib.rs @@ -54,9 +54,34 @@ pub fn engine_commit() -> &'static str { env!("STACKENCRYPT_ENGINE_COMMIT") } +#[cfg(test)] +#[path = "../engine_commit_rules.rs"] +mod engine_commit_rules; + #[cfg(test)] mod tests { use super::engine_commit; + use super::engine_commit_rules::{from_override, is_valid}; + + const SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + + /// The override rules: an empty value reads git, a value is lower-cased, + /// and only a full hex SHA, with `-dirty` or not, is a commit. + #[test] + fn the_override_is_lower_cased_and_checked() { + assert_eq!(from_override(""), None); + assert_eq!( + from_override(&SHA.to_ascii_uppercase()).as_deref(), + Some(SHA) + ); + assert!(is_valid(SHA)); + assert!(is_valid(&format!("{SHA}-dirty"))); + assert!(!is_valid(&SHA[..39])); + assert!(!is_valid(&format!("{SHA}0"))); + assert!(!is_valid(&SHA.replace('a', "g"))); + assert!(!is_valid(&SHA.to_ascii_uppercase())); + assert!(!is_valid(&format!("{SHA}-clean"))); + } use std::process::Command; #[test] diff --git a/languages/python/crates/core/tests/gate_frame_reader.rs b/languages/python/crates/core/tests/gate_frame_reader.rs index 13ae7196b..89dfa44ec 100644 --- a/languages/python/crates/core/tests/gate_frame_reader.rs +++ b/languages/python/crates/core/tests/gate_frame_reader.rs @@ -9,7 +9,7 @@ //! does not check the bytes of a real decrypt result. It does not decrypt: //! it proves the reading half, which is the half the extension owns. -use stack_encrypt::dynamic::frame::Frame; +use stack_encrypt::dynamic::frame::{Frame, FrameWriter}; use stack_encrypt::dynamic::plaintext::{ Encrypt, Error, FixedType, Int, Logical, MapSink, Physical, SeqSink, Sink, SourceError, Width, }; @@ -237,6 +237,55 @@ fn an_outside_sink_reads_a_typed_null_a_sequence_and_a_nested_map() { ); } +/// A batch of two records, written the way `dynamic::decrypt` writes one +/// (`write_record` in stack-encrypt's `record.rs`): a plain `FrameWriter`, +/// a sequence, one map per record, and each field's `ValueFrame` walked into +/// its entry. The reader must see the same fields as in a `ValueFrame::of` +/// record, once for each record. +#[test] +fn an_outside_sink_reads_a_batch_written_as_decrypt_writes_it() { + let record = |email: &str, age: i64| { + vec![ + ("email", ValueFrame::of(email).expect("a str frames")), + ("age", ValueFrame::of(age).expect("an i64 frames")), + ] + }; + let rows = [ + record("ada@example.com", 36), + record("grace@example.com", 85), + ]; + + let mut bytes = Vec::new(); + let mut seq = FrameWriter::new(&mut bytes) + .seq(Some(rows.len())) + .expect("a sequence starts"); + for fields in rows { + let mut map = seq + .element() + .expect("an element") + .map(Some(fields.len())) + .expect("a map starts"); + for (name, value) in fields { + value + .visit(map.entry(name).expect("an entry")) + .expect("a field walks"); + } + map.end().expect("a map ends"); + } + seq.end().expect("the sequence ends"); + + let frame = Frame::parse(&bytes).expect("the batch is one frame"); + assert_eq!( + read(frame), + vec![ + seen("email", None, "text", false), + seen("age", None, "int", false), + seen("email", None, "text", false), + seen("age", None, "int", false), + ], + ); +} + #[test] fn malformed_frame_bytes_are_refused_before_any_walk() { let record = ValueFrame::of(User).expect("the record is in the value model"); diff --git a/languages/python/crates/engine/src/values.rs b/languages/python/crates/engine/src/values.rs index 524152e11..79be62975 100644 --- a/languages/python/crates/engine/src/values.rs +++ b/languages/python/crates/engine/src/values.rs @@ -472,4 +472,97 @@ mod tests { fn decimal_of(value: &Bound<'_, PyAny>) -> Result, Refusal> { decimal(value, "decimal").unwrap() } + + /// A positive exponent multiplies the mantissa, and the sign applies to + /// it. A wrong sign or power would encrypt another number with no error. + #[test] + fn a_decimal_reads_as_its_exact_mantissa_and_scale() { + Python::initialize(); + Python::attach(|py| { + let decimal = py.import("decimal").unwrap().getattr("Decimal").unwrap(); + for (text, expected_mantissa, expected_scale) in [ + ("1.2E+3", 1_200, 0), + ("-0.05", -5, 2), + ("-7", -7, 0), + ("1E+28", 10_i128.pow(28), 0), + ] { + match decimal_of(&decimal.call1((text,)).unwrap()) { + Ok(Value::Decimal { mantissa, scale }) => { + assert_eq!( + (mantissa, scale), + (expected_mantissa, expected_scale), + "{text}" + ); + } + other => panic!("{text}: {other:?}"), + } + } + }); + } + + /// A date is days from 1970-01-01, and a datetime nanoseconds from the + /// epoch, in UTC. Both are checked against known numbers, before the + /// epoch and with a non-UTC offset, and read back as equal values. + #[test] + fn a_date_and_a_datetime_read_as_the_engines_numbers_and_back() { + Python::initialize(); + Python::attach(|py| { + let datetime = py.import("datetime").unwrap(); + let utc = datetime + .getattr("timezone") + .unwrap() + .getattr("utc") + .unwrap(); + let ten_hours = datetime + .getattr("timezone") + .unwrap() + .call1((datetime + .getattr("timedelta") + .unwrap() + .call1((0, 36_000)) + .unwrap(),)) + .unwrap(); + let day_before = datetime + .getattr("date") + .unwrap() + .call1((1969, 12, 31)) + .unwrap(); + let cases = [ + (day_before, PyKind::Date, Scalar::Date(-1)), + ( + // 1970-01-01T00:00:00.123456Z, written ten hours ahead. + datetime + .getattr("datetime") + .unwrap() + .call1((1970, 1, 1, 10, 0, 0, 123_456, &ten_hours)) + .unwrap(), + PyKind::DateTime, + Scalar::Timestamp(123_456_000), + ), + ( + datetime + .getattr("datetime") + .unwrap() + .call1((1969, 12, 31, 23, 59, 59, 999_999, &utc)) + .unwrap(), + PyKind::DateTime, + Scalar::Timestamp(-1_000), + ), + ]; + for (value, kind, scalar) in cases { + let read = read(&value, kind, "test").unwrap().unwrap(); + let got = match read.value() { + Value::Date(days) => Scalar::Date(i32::try_from(days).unwrap()), + Value::Timestamp(nanos) => Scalar::Timestamp(nanos), + other => panic!("{value}: read as {other:?}"), + }; + assert_eq!(got, scalar, "{value}"); + let opened = to_python(py, &scalar).unwrap().unwrap(); + assert!( + opened.bind(py).eq(&value).unwrap(), + "{value} came back as {opened}" + ); + } + }); + } } diff --git a/packages/eql/crates/eql-codegen/src/py_eql.rs b/packages/eql/crates/eql-codegen/src/py_eql.rs index 4a074273b..069933e7c 100644 --- a/packages/eql/crates/eql-codegen/src/py_eql.rs +++ b/packages/eql/crates/eql-codegen/src/py_eql.rs @@ -64,15 +64,40 @@ fn py_tuple(items: &[&str]) -> String { /// The class docstring: the summary and the status line the Go emitter /// writes too ([`Row::summary`], [`Row::status`]). fn docstring(row: &Row) -> String { - let text = format!( + py_docstring(&format!( "The EQL type {summary}.\n\n {status}\n ", summary = row.summary(), status = row.status(), - ); + )) +} + +/// `text` as a Python docstring literal. A backslash is doubled, so Python +/// reads it as written. Text with `"""` in it is refused: it would end the +/// docstring. +fn py_docstring(text: &str) -> String { assert!(!text.contains("\"\"\""), "catalog text ends the docstring"); format!("\"\"\"{}\"\"\"", text.replace('\\', "\\\\")) } +/// The names a catalog row may not take as its class name: Python keywords +/// that are names, and the module's own names, which a class of the same +/// name would replace. +const RESERVED_NAMES: [&str; 5] = ["Marker", "None", "True", "False", "TYPES"]; + +/// `name` as a Python class name. A catalog name is written straight into +/// `class {name}(Marker):` and into `TYPES`, so a name that is not a plain +/// class name, or that replaces one of the module's names, is refused here +/// rather than in the Python job. +fn py_ident(name: &str) -> &str { + assert!( + name.starts_with(|c: char| c.is_ascii_uppercase()) + && name.chars().all(|c| c.is_ascii_alphanumeric()) + && !RESERVED_NAMES.contains(&name), + "catalog name {name:?} is not a usable Python class name" + ); + name +} + fn class(out: &mut String, row: &Row) { let (query, query_sql_domain) = match &row.query { Some((query, domain)) => (Some(query.as_str()), Some(domain.as_str())), @@ -91,7 +116,7 @@ fn class(out: &mut String, row: &Row) { query_sql_domain = {query_sql_domain}\n \ producible = {producible}\n \ reason = {reason}\n", - name = row.name, + name = py_ident(&row.name), doc = docstring(row), name_lit = py_str(&row.name), family = py_str(row.family), @@ -136,7 +161,7 @@ pub fn render_py_eql() -> String { } out.push_str("\n\nTYPES = (\n"); for row in &rows { - let _ = writeln!(out, " {},", row.name); + let _ = writeln!(out, " {},", py_ident(&row.name)); } out.push_str(")\n"); out @@ -161,6 +186,42 @@ mod tests { assert_eq!(py_str(r#"a "b" \c"#), r#""a \"b\" \\c""#); } + #[test] + #[should_panic(expected = "control character")] + fn a_string_with_a_control_character_is_refused() { + py_str("a\nb"); + } + + #[test] + fn an_absent_string_is_none() { + assert_eq!(py_opt(None), "None"); + assert_eq!(py_opt(Some("eq")), r#""eq""#); + } + + #[test] + fn a_docstring_doubles_a_backslash() { + assert_eq!(py_docstring(r"a \b"), r#""""a \\b""""#); + } + + #[test] + #[should_panic(expected = "ends the docstring")] + fn a_docstring_with_three_quotes_is_refused() { + py_docstring(r#"a """ b"#); + } + + #[test] + fn a_class_name_is_a_plain_capitalised_identifier() { + assert_eq!(py_ident("TextEq"), "TextEq"); + for bad in [ + "None", "True", "Marker", "TYPES", "textEq", "Text_Eq", "1Text", "", + ] { + assert!( + std::panic::catch_unwind(|| py_ident(bad)).is_err(), + "{bad:?} was accepted" + ); + } + } + #[test] fn tuples_have_python_syntax_for_every_length() { assert_eq!(py_tuple(&[]), "()"); From eb5a4ccdf009a0cc4d69c3d3d348c0edd2d5ce09 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 20:33:58 +0900 Subject: [PATCH 19/21] test(python): close the gaps the review marked low - An offline test is skipped once a live test has built the shared engine client in the same process. A combined run had 51 failures. - refused() checks that the refused value is not in the error text. - The same value encrypts to three different ciphertexts. - The declaration cases are compiled with pytest's assertion rewrite, so a failure shows its values. - New tests for four README claims: a redact=False class may replace its printing methods, decrypt refuses a uuid.UUID for a str field, and bound_names keeps only top-level names, now with except-as and a walrus in a comprehension. - The gate test says that CI does not run it, and what its timing assertion can show. Addresses review threads 1414-F1, F2, 1433-F4, 1499-F8, F10, F11, and 1517-F3, F4, F5, F6. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/tests/_compiled_cases.py | 25 +++++++++++++++++++ languages/python/tests/_readme.py | 18 +++++++++++-- languages/python/tests/conftest.py | 13 +++++++++- languages/python/tests/test_cipher.py | 13 ++++++++++ languages/python/tests/test_declaration.py | 8 +++--- .../test_declaration_future_annotations.py | 18 ++++--------- languages/python/tests/test_decrypt.py | 18 +++++++++++++ languages/python/tests/test_encrypt.py | 9 ++++++- languages/python/tests/test_encrypt_live.py | 12 +++++++++ .../tests/test_gate_keyboard_interrupt.py | 11 +++++++- languages/python/tests/test_readme.py | 25 +++++++++++++++++++ 11 files changed, 148 insertions(+), 22 deletions(-) create mode 100644 languages/python/tests/_compiled_cases.py diff --git a/languages/python/tests/_compiled_cases.py b/languages/python/tests/_compiled_cases.py new file mode 100644 index 000000000..f0f2714ad --- /dev/null +++ b/languages/python/tests/_compiled_cases.py @@ -0,0 +1,25 @@ +"""Compile ``_declaration_cases.py`` with pytest's assertion rewriting. + +The cases run through ``exec``, so pytest does not rewrite their ``assert`` +statements as it rewrites a test module's. A failure would then show a bare +``AssertionError`` with no values. This applies the same rewrite to the +parsed source first. ``rewrite_asserts`` is in pytest's private ``_pytest`` +package, so a pytest release can change it; a change fails both declaration +test modules at import. +""" + +import ast +import pathlib +from types import CodeType + +from _pytest.assertion.rewrite import rewrite_asserts + +CASES = pathlib.Path(__file__).with_name("_declaration_cases.py") + + +def compiled_cases(flags: int = 0) -> CodeType: + """The cases, with their asserts rewritten, compiled with ``flags``.""" + source = CASES.read_bytes() + tree = ast.parse(source, filename=str(CASES)) + rewrite_asserts(tree, source, str(CASES)) + return compile(tree, str(CASES), "exec", flags=flags, dont_inherit=True) diff --git a/languages/python/tests/_readme.py b/languages/python/tests/_readme.py index a3499133b..02ac1e230 100644 --- a/languages/python/tests/_readme.py +++ b/languages/python/tests/_readme.py @@ -67,13 +67,27 @@ def visit_Name(self, node: ast.Name) -> None: def visit_alias(self, node: ast.alias) -> None: self.bound.add((node.asname or node.name).split(".")[0]) + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None: + # `except E as name` binds `name`, which is a str, not an ast.Name. + if node.name: + self.bound.add(node.name) + self.generic_visit(node) + def _own_scope(self, node: ast.AST) -> None: if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): self.bound.add(node.name) + def _comprehension(self, node: ast.AST) -> None: + # A comprehension's own variables stay inside it, but a walrus + # (`:=`) in it binds its target in the enclosing scope. + for inner in ast.walk(node): + if isinstance(inner, ast.NamedExpr) and isinstance(inner.target, ast.Name): + self.bound.add(inner.target.id) + visit_FunctionDef = visit_AsyncFunctionDef = visit_ClassDef = _own_scope - visit_Lambda = visit_ListComp = visit_SetComp = visit_DictComp = _own_scope - visit_GeneratorExp = _own_scope + visit_Lambda = _own_scope + visit_ListComp = visit_SetComp = visit_DictComp = _comprehension + visit_GeneratorExp = _comprehension def _top_level(source: str) -> _TopLevel: diff --git a/languages/python/tests/conftest.py b/languages/python/tests/conftest.py index 21f0c53bc..70a4e936d 100644 --- a/languages/python/tests/conftest.py +++ b/languages/python/tests/conftest.py @@ -36,7 +36,18 @@ def no_credentials(monkeypatch, tmp_path): def credentials(monkeypatch, no_credentials): """Credentials that parse, so ``Cipher`` builds, and no client key. A call that gets past every check fails building the engine client, with - no request.""" + no request. + + The engine client is shared by the whole process. Once a live test has + built it, a call here would reach ZeroKMS instead of failing, so the + test is skipped. ``mise run test`` and ``-m live`` run the two kinds of + test in separate processes, and skip nothing. + """ + if _engine.client_builds() > 0: + pytest.skip( + "a live test built the shared engine client earlier in this " + "process; run the offline tests in a process of their own" + ) monkeypatch.setenv("CS_CLIENT_ACCESS_KEY", ACCESS_KEY) monkeypatch.setenv("CS_WORKSPACE_CRN", WORKSPACE_CRN) diff --git a/languages/python/tests/test_cipher.py b/languages/python/tests/test_cipher.py index fdd7f40be..a3b619136 100644 --- a/languages/python/tests/test_cipher.py +++ b/languages/python/tests/test_cipher.py @@ -153,6 +153,19 @@ def test_each_replaced_method_is_refused(credentials, name): assert f"{name} replaced" in str(error) +def test_a_redact_false_class_may_replace_its_printing_methods(credentials): + # The README says Cipher checks the printing methods only when the + # decorator replaced them. + @stackencrypt.model("users", redact=False) + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + + User.__repr__ = lambda self: "replaced" + + Cipher(User) + + def test_a_hint_that_no_longer_resolves_names_the_name(credentials): User = declared() User.__annotations__["posts"] = "List[Undefined]" diff --git a/languages/python/tests/test_declaration.py b/languages/python/tests/test_declaration.py index 4ef9304f1..3506cb3ce 100644 --- a/languages/python/tests/test_declaration.py +++ b/languages/python/tests/test_declaration.py @@ -1,10 +1,10 @@ """The declaration tests, with annotations evaluated as written. The tests themselves are in ``_declaration_cases.py``, compiled here without -``from __future__ import annotations``. +``from __future__ import annotations``, with their asserts rewritten as +pytest rewrites a test module's (``_compiled_cases.py``). """ -import pathlib +from _compiled_cases import compiled_cases -_CASES = pathlib.Path(__file__).with_name("_declaration_cases.py") -exec(compile(_CASES.read_text(), str(_CASES), "exec", dont_inherit=True), globals()) +exec(compiled_cases(), globals()) diff --git a/languages/python/tests/test_declaration_future_annotations.py b/languages/python/tests/test_declaration_future_annotations.py index 3d9d4f5a7..57dd04403 100644 --- a/languages/python/tests/test_declaration_future_annotations.py +++ b/languages/python/tests/test_declaration_future_annotations.py @@ -1,20 +1,12 @@ """The declaration tests, compiled with ``from __future__ import annotations``. Under that import every annotation is a string until the decorator resolves -it, which is the case a real model module with the import is in. +it, which is the case a real model module with the import is in. The asserts +are rewritten as pytest rewrites a test module's (``_compiled_cases.py``). """ import __future__ -import pathlib -_CASES = pathlib.Path(__file__).with_name("_declaration_cases.py") -exec( - compile( - _CASES.read_text(), - str(_CASES), - "exec", - flags=__future__.annotations.compiler_flag, - dont_inherit=True, - ), - globals(), -) +from _compiled_cases import compiled_cases + +exec(compiled_cases(__future__.annotations.compiler_flag), globals()) diff --git a/languages/python/tests/test_decrypt.py b/languages/python/tests/test_decrypt.py index 01e1dd190..2bba9b707 100644 --- a/languages/python/tests/test_decrypt.py +++ b/languages/python/tests/test_decrypt.py @@ -9,6 +9,7 @@ import dataclasses import datetime import json +import uuid from typing import Annotated import pytest @@ -189,6 +190,23 @@ def test_a_passthrough_int_that_arrives_as_str_is_the_wrong_type(untouched): assert error.reason == "passthrough" +@stackencrypt.model("keyed") +@dataclasses.dataclass +class Keyed: + email: Annotated[str, TextEq] + id: Annotated[str, Passthrough] + + +def test_a_uuid_for_a_str_passthrough_is_the_wrong_type(untouched): + # The README's UUID advice depends on this: psycopg reads a uuid column + # as a uuid.UUID, and decrypt does not convert it to str. + stored = json.dumps({"i": {"t": "keyed", "c": "email"}, "v": 2}) + with pytest.raises(InvalidValueError) as raised: + Cipher(Keyed).decrypt([{"email": stored, "id": uuid.uuid4()}]) + assert raised.value.code == "value_wrong_type" + assert raised.value.field == "id" + + def test_a_null_passthrough_is_refused(untouched): error = refused([{"email": STORED, "age": None}], InvalidValueError) assert error.code == "value_none" diff --git a/languages/python/tests/test_encrypt.py b/languages/python/tests/test_encrypt.py index 7e5ff5cb6..2978559d8 100644 --- a/languages/python/tests/test_encrypt.py +++ b/languages/python/tests/test_encrypt.py @@ -44,10 +44,17 @@ def cipher_for(**fields): def refused(cipher, model): + """The refusal of ``model``. Neither the valid email nor the refused + value may appear in it. None and a bool are not checked: their text can + appear in a message for another reason.""" with pytest.raises(InvalidValueError) as raised: cipher.encrypt([model]) error = raised.value assert SECRET not in str(error) + value = getattr(model, error.field) if error.field else None + if value is not None and not isinstance(value, bool): + for shown in {repr(value), str(value)}: + assert shown not in str(error), f"{shown} is in {error}" return error @@ -102,7 +109,7 @@ def test_a_passthrough_value_of_another_type_is_refused(untouched): def test_an_int_outside_its_kind_is_out_of_range(untouched): cipher, User = cipher_for(rank=Annotated[int, Smallint]) - error = refused(cipher, User(email=SECRET, rank=40_000)) + error = refused(cipher, User(email=SECRET, rank=40_123)) assert error.code == "value_out_of_range" assert error.field == "rank" assert error.reason == "int16" diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py index 82bf0c06f..3c6d7d627 100644 --- a/languages/python/tests/test_encrypt_live.py +++ b/languages/python/tests/test_encrypt_live.py @@ -117,6 +117,18 @@ def test_1001_one_field_models_make_three_key_requests(): assert _engine.request_count() - before == 3 +def test_the_same_value_encrypts_to_a_different_ciphertext_each_time(): + # A reused nonce, or a deterministic seal, would give equal payloads. + users = Cipher(User) + model = User(email="same@example.com", age=3) + + first, second = users.encrypt([model, model]) + again = users.encrypt([model])[0] + + ciphertexts = {json.loads(row["email"])["c"] for row in (first, second, again)} + assert len(ciphertexts) == 3 + + def test_two_ciphers_share_one_engine_client(): Cipher(User).encrypt([User(email="a@example.com", age=1)]) # Read after the first call: an earlier test may have loaded the keyset diff --git a/languages/python/tests/test_gate_keyboard_interrupt.py b/languages/python/tests/test_gate_keyboard_interrupt.py index 93a78b271..6ae01cd48 100644 --- a/languages/python/tests/test_gate_keyboard_interrupt.py +++ b/languages/python/tests/test_gate_keyboard_interrupt.py @@ -2,7 +2,16 @@ the GIL is raised only after that call returns. Runs only against a wheel built with the `gate-probe` cargo feature; see -docs/gate-facts/keyboard-interrupt.md. Skipped otherwise. +docs/gate-facts/keyboard-interrupt.md. Skipped otherwise, and CI builds no +such wheel. So this is a check run by hand when the detach or the signal +handling changes, not one CI keeps. + +What the first test shows: Python runs a signal handler only on the main +thread, between bytecodes. So the interrupt cannot be raised while the main +thread is inside `probe()`, whether or not the call released the GIL. The +timing assertion therefore holds by the interpreter's design. What it does +show is that the call returns normally, and that the interrupt is raised +after it, with the extension still usable. """ import sys diff --git a/languages/python/tests/test_readme.py b/languages/python/tests/test_readme.py index fc57f6986..a12ff7ab2 100644 --- a/languages/python/tests/test_readme.py +++ b/languages/python/tests/test_readme.py @@ -26,6 +26,31 @@ def test_the_python_blocks_bind_the_names_the_live_check_reads(): assert check_names() <= bound_names(program()) +def test_bound_names_keeps_only_the_top_level_names(): + source = ( + "import os.path as p\n" + "top = 1\n" + "def f():\n" + " inside_function = 2\n" + "class C:\n" + " inside_class = 3\n" + "squares = [n * n for n in range(3)]\n" + "lam = lambda x: x\n" + ) + assert bound_names(source) == {"p", "top", "f", "C", "squares", "lam"} + + +def test_bound_names_reads_except_as_and_a_walrus_in_a_comprehension(): + source = ( + "try:\n" + " pass\n" + "except ValueError as caught:\n" + " pass\n" + "found = [last := n for n in range(3)]\n" + ) + assert bound_names(source) == {"caught", "found", "last"} + + def test_the_readme_has_sql_for_the_live_test_to_run(): assert blocks("sql") From 47bbb9fccc4a87b36a710d37fbf8cca539c35970 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 20:34:27 +0900 Subject: [PATCH 20/21] ci(python): cache both workspaces in one step, and check two more steps The live job had one rust-cache step per workspace. Each pruned the shared cargo registry to its own crates before it saved, so each save lost the other's. One step now lists both. eql-suite-ci.test.mjs reads workspaces: as a list. python-ci.test.mjs now checks that the workflow runs the _eql.py parity test, and that the build task installs from uv.lock with --locked, --no-deps and --no-build-isolation. Addresses review threads 1415-F1, 1499-F16, and 1582-F2. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests-python.yml | 14 +++---- scripts/__tests__/eql-suite-ci.test.mjs | 53 ++++++++++++++++--------- scripts/__tests__/python-ci.test.mjs | 51 ++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 25 deletions(-) diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index e69b3039b..13511e9ba 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -277,15 +277,15 @@ jobs: # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs cache_key_prefix: mise-v1-ubuntu-2404 - # One cache per workspace: this one, and EQL's, whose eql-codegen - # writes the bundle. + # One step for both workspaces: this one, and EQL's, whose eql-codegen + # writes the bundle. Two steps would each prune the shared cargo + # registry to their own crates before they save, so each save would + # lose the other workspace's crates. - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: - workspaces: languages/python - - - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - with: - workspaces: packages/eql + workspaces: | + languages/python + packages/eql # The builds, with no credentials: the extension into .venv, and the # EQL v4 bundle. diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index 8941c20e9..79755e0b4 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -586,23 +586,34 @@ const RUST_CACHE_STEPS = workflowFiles().flatMap((relPath) => { }) /** - * Whether a rust-cache step names the root of another Cargo workspace: a - * directory outside the EQL tree with its own `Cargo.lock`. A job that builds - * both EQL and that workspace (tests-python.yml's `live` builds the Python - * wheel and the EQL bundle) caches each in its own step, and the other - * workspace's step is right where it is. A member of the EQL workspace, or - * the monorepo root, is not another workspace, so this check still catches - * both. + * The workspaces a rust-cache step names: one per line of `workspaces:`, as + * rust-cache reads it. A job that builds EQL and another workspace + * (tests-python.yml's `live` builds the Python wheel and the EQL bundle) + * lists both in one step. Two steps would each prune the shared cargo + * registry to their own crates before they save. */ +const workspacesOf = (step) => + String(step?.with?.workspaces ?? '') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + +/** + * Whether `dir` is the root of another Cargo workspace: a directory outside + * the EQL tree with its own `Cargo.lock`. A member of the EQL workspace, or + * the monorepo root, is not another workspace, so the check below still + * catches both. + */ +const isAnotherWorkspace = (dir) => + dir !== '.' && + dir !== CARGO_WORKSPACE && + !dir.startsWith(`${CARGO_WORKSPACE}/`) && + existsSync(join(REPO_ROOT, dir, 'Cargo.lock')) + +/** A step that caches other workspaces only, and not EQL's. */ const namesAnotherWorkspace = ({ step }) => { - const dir = step?.with?.workspaces - return ( - typeof dir === 'string' && - dir !== '.' && - dir !== CARGO_WORKSPACE && - !dir.startsWith(`${CARGO_WORKSPACE}/`) && - existsSync(join(REPO_ROOT, dir, 'Cargo.lock')) - ) + const dirs = workspacesOf(step) + return dirs.length > 0 && dirs.every(isAnotherWorkspace) } /** The rust-cache steps in EQL's jobs that cache the EQL workspace. */ @@ -650,9 +661,15 @@ describe('the shared Rust cache is pointed and saved correctly', () => { // root workspace, and rust-cache's default (`. -> target`) is right there. // Held non-empty, so a change to `isEqlJob` cannot empty the check. expect(EQL_RUST_CACHE_STEPS.length).toBeGreaterThan(0) - const wrong = EQL_RUST_CACHE_STEPS.filter( - ({ step }) => step?.with?.workspaces !== CARGO_WORKSPACE, - ).map((entry) => + // The step names the EQL root, and any other entry is another + // workspace's root, never an EQL member or the monorepo root. + const wrong = EQL_RUST_CACHE_STEPS.filter(({ step }) => { + const dirs = workspacesOf(step) + return ( + !dirs.includes(CARGO_WORKSPACE) || + !dirs.every((dir) => dir === CARGO_WORKSPACE || isAnotherWorkspace(dir)) + ) + }).map((entry) => stepLabel(entry, `workspaces: ${entry.step?.with?.workspaces}`), ) expect( diff --git a/scripts/__tests__/python-ci.test.mjs b/scripts/__tests__/python-ci.test.mjs index 9b5df3f85..2ff648195 100644 --- a/scripts/__tests__/python-ci.test.mjs +++ b/scripts/__tests__/python-ci.test.mjs @@ -254,6 +254,57 @@ describe('tests-python.yml', () => { ) }) + it('runs the parity test that refuses a hand edit to _eql.py', () => { + // The one check that fails when someone edits the generated + // `_eql.py` by hand: test-eql.yml does not start on a change under + // languages/python. + const wf = readWorkflow(WORKFLOW) + const runs = Object.values(wf.jobs) + .flatMap((job) => job.steps) + .map((step) => step.run ?? '') + .filter((run) => + [ + 'cargo test', + '--locked', + '-p eql-codegen', + '--test py_eql_parity', + ].every((part) => run.includes(part)), + ) + expect( + runs, + "steps that run eql-codegen's py_eql_parity test", + ).toHaveLength(1) + }) + + it('builds from uv.lock, with no tool resolved outside it', () => { + // `uv sync --locked` refuses a lock that does not match pyproject.toml; + // without it, uv would update the lock and the build would pass. Each + // `uv pip install` must take nothing from the index: `--no-deps` for + // the package's dependencies, `--no-build-isolation` for maturin. + expect(existsSync(join(REPO_ROOT, PYTHON_ROOT, 'uv.lock'))).toBe(true) + const miseToml = read(`${PYTHON_ROOT}/mise.toml`) + const build = miseToml.match( + /^\[tasks\.build\]\n([\s\S]*?)(?=^\[|(?![\s\S]))/m, + ) + expect(build, 'mise.toml has a [tasks.build]').not.toBeNull() + const lines = build[1] + .split('\n') + .filter((line) => !line.trim().startsWith('#')) + const syncs = lines.filter((line) => /\buv sync\b/.test(line)) + expect(syncs, 'uv sync lines in the build task').toHaveLength(1) + expect(syncs[0]).toContain('--locked') + expect(syncs[0]).toContain('--group build') + const installs = lines.filter((line) => /\buv pip install\b/.test(line)) + expect( + installs.length, + 'uv pip install lines in the build task', + ).toBeGreaterThan(0) + for (const line of installs) { + expect(line).toContain('--no-deps') + expect(line).toContain('--no-build-isolation') + } + }) + it('runs every task of the Python workspace, and the two cargo lints, from that folder', () => { // "A check nothing invokes reads exactly like a check that passes" // (crates-ci.test.mjs). That file scans the root mise.toml and the crate From bf84840af1524efe24f7520496d821c964622d06 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 20:45:55 +0900 Subject: [PATCH 21/21] docs(python): close the example's first four findings The customer directory example recorded four gaps in the README: - Pydantic's ValidationError and FastAPI's 422 response show the rejected input. The README's list of what shows plaintext now names both. hide_input_in_errors hides the input from the error's text, but not from errors(), which FastAPI's response is built from. - A key the database generates cannot be a required field. The UUID key section now covers it, for psycopg and for the SQLAlchemy layer. - decrypt keeps the order of the rows. The README and the docstring now say so, and a deterministic test checks it with the rows reversed. - A unique index on eq_term works because equal values have equal eq terms. The README now says so. Each finding in FINDINGS.md now has a status line. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 8 ++++++-- .../examples/customer-directory/FINDINGS.md | 10 +++++++++- .../python/python/stackencrypt/_cipher.py | 2 ++ .../python/tests_deterministic/test_order.py | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 languages/python/tests_deterministic/test_order.py diff --git a/languages/python/README.md b/languages/python/README.md index 9421e26eb..6b979ff1d 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -81,7 +81,7 @@ with psycopg.connect(DATABASE_URL) as conn: users = cipher.decrypt(cur.fetchall()) # a list of User ``` -`decrypt` builds whole `User` instances, so the `SELECT` must name every declared column, passthrough columns included. A row without one raises `stackencrypt.DecryptError` with the code `decrypt_missing_column`. Other columns in the row are ignored. +`decrypt` returns one `User` for each row, in the order of the rows. `decrypt` builds whole `User` instances, so the `SELECT` must name every declared column, passthrough columns included. A row without one raises `stackencrypt.DecryptError` with the code `decrypt_missing_column`. Other columns in the row are ignored. A passthrough `datetime` field needs a `timestamptz` column. psycopg reads a `timestamp` column as a naive `datetime`, with no `tzinfo`. `decrypt` refuses a naive `datetime` with `value_naive_datetime`, as `encrypt` does. @@ -217,7 +217,7 @@ Each class must be declared with `stackencrypt.model` and mapped by SQLAlchemy. - `declaration_table_mismatch`: the table in `stackencrypt.model` is not the `__tablename__`. - `declaration_column_name`: an encrypted field's column has another name than the field, as with `mapped_column("email_address")`. - `declaration_column_type`: an encrypted field's column is not the EQL type of its marker. Set `type_annotation_map = eql_type_map` on the base. -- `declaration_unique_column`: an encrypted column has `unique=True` or a unique constraint. Each EQL value contains new random ciphertext, so the constraint would never find a duplicate. Use a unique index on `eql_v4.eq_term(col)`, as in "Migrations with Alembic". +- `declaration_unique_column`: an encrypted column has `unique=True` or a unique constraint. Each EQL value contains new random ciphertext, so the constraint would never find a duplicate. Equal values have equal `eq` terms, so use a unique index on `eql_v4.eq_term(col)`, as in "Migrations with Alembic". - `declaration_already_configured`: another class is configured for the same table. A statement that puts an encrypted value inside its SQL, such as `insert(Customer).values(email=...)`, raises `DeclarationError` with `declaration_inline_value`. The value is not stored. Pass the values as parameters instead, as `session.add` and `session.execute(insert(Customer), rows)` do. @@ -326,6 +326,8 @@ Nothing else is hidden. These all contain the plaintext: - a log line that formats a field, such as `log.info(user.email)` - `dataclasses.asdict`, Pydantic's `model_dump`, `vars`, and `__dict__` - `pickle.dumps(user)`, and anything a debugger reads from the attributes +- a Pydantic `ValidationError`, which shows the rejected input. Set `hide_input_in_errors=True` in the model's `model_config` to leave it out. +- FastAPI's 422 response, which repeats the input of each error. `hide_input_in_errors` does not change it. Replace FastAPI's handler for `RequestValidationError` with one that leaves out `input`. If the class defines its own `__repr__`, `__str__`, or `__rich_repr__`, the decorator refuses it. `__rich_repr__` is the method the `rich` library prints through. To keep your own methods, pass `redact=False` to `stackencrypt.model`. Your methods must then hide the plaintext themselves. @@ -342,6 +344,8 @@ A model field cannot be a `uuid.UUID`. The decorator refuses it with the code `d - Leave the key column out of the model. `encrypt` never writes a column the model does not declare, and `decrypt` ignores one. Insert and select the key yourself, beside the encrypted values. - Declare the field as `Annotated[str, Passthrough]`. Convert with `str(key)` before `encrypt`. Select the column as `id::text`, because psycopg reads a `uuid` column as a `uuid.UUID`, and `decrypt` refuses that for a `str` field. +A key the database generates, such as an `identity` column, cannot be a required field: the model is encrypted before PostgreSQL creates the key. Leave it out of the model, or declare it `Annotated[Optional[int], Ignore] = None`. Insert with `RETURNING id` to get the new key. To read it back, select it beside the encrypted columns. `decrypt` returns the instances in the order of the rows, so the key of row `n` belongs to instance `n`. With the SQLAlchemy layer, declare it `Mapped[Annotated[int, Ignore]]` with `mapped_column(primary_key=True)`, as in that section's example. + ## What is not supported yet - Range queries with `between`. Use `gte` and `lte` together. diff --git a/languages/python/examples/customer-directory/FINDINGS.md b/languages/python/examples/customer-directory/FINDINGS.md index 15c43a5f5..113d1cb89 100644 --- a/languages/python/examples/customer-directory/FINDINGS.md +++ b/languages/python/examples/customer-directory/FINDINGS.md @@ -4,7 +4,7 @@ Measured on 9 November 2026, on one macOS machine, against real ZeroKMS. The SDK ## Gaps in the SDK or its documentation -Items 5, 6, and 7 were found before the SQLAlchemy layer existed. Each now has a status line. +Items 5, 6, and 7 were found before the SQLAlchemy layer existed. Items 1 to 7 each now have a status line. ### 1. The README does not warn about Pydantic's `ValidationError` @@ -16,6 +16,8 @@ The example sets `hide_input_in_errors=True` and replaces FastAPI's 422 response Suggested action: add both to the README's list, with the setting that hides the input. +Status: addressed. The README's list in "What a printed model shows" now names both. It gives `hide_input_in_errors=True` for the `ValidationError`, and says that FastAPI's 422 response needs its own handler: with that setting, `errors()` still contains the input (checked with Pydantic 2.14.3). + ### 2. The README has no recipe for a database-generated key The README covers a UUID key only. A required integer `id` on the model cannot work, because the model is encrypted before PostgreSQL creates the `id`. @@ -24,6 +26,8 @@ The example leaves `id` out of the model. It works, but the model then cannot re Suggested action: add the database-generated case to "Store a UUID key". +Status: addressed. "Store a UUID key" now covers a key the database generates, for psycopg and for the SQLAlchemy layer. + ### 3. `decrypt` does not say whether it keeps row order Neither the `decrypt` docstring nor the README says that results come back in the order of the input rows. An application that needs a column outside the model must pair it with the decrypted result by position. @@ -32,6 +36,8 @@ The example avoids this by not returning `id`. I did not test the order. Suggested action: state the guarantee, and add a test for it. +Status: addressed. The README and the `Cipher.decrypt` docstring state that the instances come back in the order of the rows. `tests_deterministic/test_order.py` checks it, with the rows in reverse order. + ### 4. The README does not cover uniqueness on an encrypted column The README shows only a hash index, and PostgreSQL cannot make a hash index unique. This B-tree index works: @@ -44,6 +50,8 @@ A second insert with the same email raises `UniqueViolation` with SQLSTATE 23505 Suggested action: document the unique index, and say that equal emails produce equal terms. +Status: addressed. The SQLAlchemy section of the README explains `declaration_unique_column`. It says that equal values have equal `eq` terms, and shows a unique index on `eql_v4.eq_term(email)`. + ### 5. The README has no write or read path for SQLAlchemy The README shows how to declare a SQLAlchemy model, including relationships. It does not show an insert or a query. `app_sqlalchemy.py` needed three things that the README does not mention: diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index e6900ddf2..1e6817ce6 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -219,6 +219,8 @@ def decrypt(self, rows: Sequence[Mapping[str, object]]) -> List[T]: field's raises ``InvalidValueError``; it is never converted. The call is all or nothing: no instance is built when any row fails. + It returns one instance for each row, in the order of the rows. + Each of these errors names the row. Two errors from the engine name the field only, not the row: ``decrypt_malformed`` for a value that is not the field's EQL type, and ``decrypt_kind_mismatch`` for a diff --git a/languages/python/tests_deterministic/test_order.py b/languages/python/tests_deterministic/test_order.py new file mode 100644 index 000000000..fc14d7b26 --- /dev/null +++ b/languages/python/tests_deterministic/test_order.py @@ -0,0 +1,18 @@ +"""``decrypt`` returns one instance per row, in the order of the rows. An app +that selects a column outside the model, such as a generated key, pairs it +with the instances by position, as the README says. +""" + +from _sync_contract import User, user + +from stackencrypt import Cipher + + +def test_decrypt_returns_the_instances_in_the_order_of_the_rows(): + cipher = Cipher(User) + rows = cipher.encrypt([user(n) for n in range(6)]) + + assert cipher.decrypt(rows[::-1]) == [user(n) for n in reversed(range(6))] + assert cipher.decrypt_rows(rows[::2]) == [ + {"id": n, "email": user(n).email} for n in range(0, 6, 2) + ]