diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 52609d094a..a7908e8834 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -83,6 +83,11 @@ exclude_re = [ # `Ok(())`, so the replacement is the body. Text overrides it with # `query::order_text`, pinned by `an_ordering_refuses_only_text_it_cannot_encode`. 'stack-encrypt/src/sem/mod\.rs:\d+:\d+: replace sealed::Sealed::check_query -> Result<\(\), super::TermError> with Ok\(\(\)\)$', + # A JSON document type's default options are the defaults: the trait's + # body is `JsonOptions::default()`. A type that overrides them is pinned + # by `a_field_that_names_no_options_queries_under_its_types_own`, and + # `Option` forwarding by `a_nullable_field_queries_under_its_types_own_options`. + 'stack-encrypt/src/json/target\.rs:\d+:\d+: replace JsonQueries::default_options -> JsonOptions with Default::default\(\)$', # Both unit-context conversions explicitly return Self::default(). 'stack-encrypt/src/target/context\.rs:\d+:\d+: replace for (DeclaredContext|ExpectedContext)>::from -> Self with Default::default\(\)$', # A caller shape with no placeholder IS `CallerShape::default()`: diff --git a/.changeset/eql-json-field-marker-queries.md b/.changeset/eql-json-field-marker-queries.md new file mode 100644 index 0000000000..9aa551952e --- /dev/null +++ b/.changeset/eql-json-field-marker-queries.md @@ -0,0 +1,12 @@ +--- +'@cipherstash/eql': minor +--- + +`eql-bindings`, the Rust crate released with this package: a `JsonTreeDocument` field of a derived Stack Encrypt record is queried from its field marker, and each query returns the EQL query type a parameter binds as. + +- `EncryptedCustomer::PROFILE.contains(doc)` gives a `JsonTreeQuery` for `@>`, and `contained_by(doc)` one for `<@`. +- `at(path).gt(v)` and the other comparisons give a `JsonTreeLeafQuery`, `at(path).between(lo, hi)` two of them, and `at(path).selector()` the `JsonTreeSelector` that `->` takes. +- `elements(path).eq(member, v)..` gives a `JsonTreeElementsQuery`, `predicate(text)` a `JsonTreePredicateQuery`, and `children(path)` a `JsonTreeToken`. +- `#[stash(json(ordered = ["$.age"]))]` on the field names the paths the documents order. The field is sealed under those options and queried under them. + +Each query is the operand a data plan naming `JsonSearch` derives for the same column and options, which is what the Go SDK sends. The stored format, the SQL bundle and the TypeScript types do not change. diff --git a/.changeset/eql-remove-structural-traits.md b/.changeset/eql-remove-structural-traits.md index a2fef48279..17a5a993e9 100644 --- a/.changeset/eql-remove-structural-traits.md +++ b/.changeset/eql-remove-structural-traits.md @@ -6,7 +6,7 @@ - Every type keeps `EncryptTarget` and `DecryptSource`. Write with `value.encrypt_into_with_context(&keyset, column)` and read with `P::decrypt_from_with_context(stored, &keyset, column)`. - A typed query is a field marker of a derived record, such as `EncryptedMember::EMAIL.eq("bob").encrypt(&keyset)`. -- A JSON document is a `JsonPlaintext`, not a `serde_json::Value`. A typed JSON query has no replacement yet: derive the operand through a data plan that names `JsonSearch`. +- A JSON document is a `JsonPlaintext`, not a `serde_json::Value`. A typed JSON query is a document field's marker query, such as `EncryptedCustomer::PROFILE.at("$.age").gt(30).encrypt(&keyset)`. - `JsonTree` no longer implements `stack_encrypt::target::Layout`. `encryption()`, `index()` and `specs()` are its own methods. The stored format, the SQL bundle, and the TypeScript types do not change. diff --git a/docs/plans/2026-10-24-stack-encrypt-api-plan.md b/docs/plans/2026-10-24-stack-encrypt-api-plan.md index 89ff2f9d72..e67e919313 100644 --- a/docs/plans/2026-10-24-stack-encrypt-api-plan.md +++ b/docs/plans/2026-10-24-stack-encrypt-api-plan.md @@ -421,30 +421,32 @@ the strict checks on JSON input. A program can also encrypt a borrowed any [borrowed value](#encrypt-borrowed-values). With the default options, no path has an ordering term. So a query cannot -compare the order of the values at a path. To order some paths, declare a -named type with typed Rust options: +compare the order of the values at a path. To order some paths, a record +field names them with `#[stash(json(..))]`, as Go's struct tag does +(`encrypt_into=JsonSearch(ordered=$.age)`). Decided by Dan on 16 November +2026, in place of a named type with options in a `target(..)` attribute: ```rust -use stack_encrypt::json::JsonOptions; - -fn profile_options() -> Result { - JsonOptions::new().ordering(["$.age", "$.visits[*].at"]) +#[derive(Zeroize)] +struct Customer { + profile: JsonPlaintext, } -// Not on `next` yet: a JSON target with typed options. #[derive(EncryptTarget, DecryptSource)] -#[stash( - plaintext = JsonPlaintext, - transparent, - target(column = ("customers", "profile"), options = profile_options) -)] -struct EncryptedProfile(JsonTreeDocument); - -let profile = JsonPlaintext::parse(r#"{"name":"Ada","age":35}"#)?; -let encrypted: EncryptedProfile = profile.encrypt_into(&keyset).await?; -let opened = JsonPlaintext::decrypt_from(encrypted, &keyset).await?; +#[stash(struct = Customer, scope = "customers")] +struct EncryptedCustomer { + #[stash(json(ordered = ["$.age", "$.visits[*].at", "$.visits[*].place"]))] + profile: JsonTreeDocument, +} + +let customer = Customer { profile: JsonPlaintext::parse(r#"{"name":"Ada","age":35}"#)? }; +let encrypted: EncryptedCustomer = customer.encrypt_into(&keyset).await?; +let opened = Customer::decrypt_from(encrypted, &keyset).await?; ``` +The field is sealed under the options, and its marker queries under them. +`sealed_root = false` in the same list turns off the sealed root. + #### Query a JSON document A JSON query is a query on the document's field marker. The marker has @@ -461,28 +463,27 @@ declaration: The marker carries both, so the query names only its kind and its values. The output type is the EQL query type, fixed by the marker. -Decided on 7 November 2026. +Decided on 7 November 2026. Built in #1459; this example runs as +`packages/eql/tests/encryption/tests/json_tree_marker.rs`. ```rust use eql_bindings::v4::json_tree::{JsonTreeElementsQuery, JsonTreeLeafQuery, JsonTreeQuery}; -// Not on `next` yet: field markers and their query methods. The method -// names are proposals. -let ada: JsonTreeQuery = EncryptedProfile::DOCUMENT +let ada: JsonTreeQuery = EncryptedCustomer::PROFILE .contains(json!({ "name": "Ada" })) .encrypt(&keyset) .await?; -let older: JsonTreeLeafQuery = EncryptedProfile::DOCUMENT +let older: JsonTreeLeafQuery = EncryptedCustomer::PROFILE .at("$.age") .gt(30) .encrypt(&keyset) .await?; -let adults: JsonTreeLeafQuery = EncryptedProfile::DOCUMENT +let (low, high): (JsonTreeLeafQuery, JsonTreeLeafQuery) = EncryptedCustomer::PROFILE .at("$.age") .between(18, 65) .encrypt(&keyset) .await?; -let home: JsonTreeElementsQuery = EncryptedProfile::DOCUMENT +let home: JsonTreeElementsQuery = EncryptedCustomer::PROFILE .elements("$.visits[*]") .eq("place", "home") .gt("at", 15) @@ -508,10 +509,11 @@ The kinds of JSON query, the marker method for each, and what each finds: | Query | Marker method | What it finds | EQL query type | | --- | --- | --- | --- | | Containment | `contains(doc)`, `contained_by(doc)`, and `at(path).eq(v)` | Rows whose document contains the given document, or is contained by it. Equality of a scalar at a path is containment of a one-entry document, so `at(path).eq(v)` lowers to it | `JsonTreeQuery` | -| Comparison at a path | `at(path).gt(v)`, `.lt(v)`, `.between(a, b)`, and the rest | Rows where the value at an ordered path orders against, or lies between, the given values | `JsonTreeLeafQuery` | +| Comparison at a path | `at(path).gt(v)`, `.lt(v)`, `.between(a, b)`, and the rest | Rows where the value at an ordered path orders against, or lies between, the given values | `JsonTreeLeafQuery`; two for `between`, low then high | +| A path's selector | `at(path).selector()` | The selector `->` and `ORDER BY eql_v4.ord_term(col -> $1)` take | `JsonTreeSelector` | | Element match | `elements(path).eq(member, v).gt(member, v)` | Rows where one element of an array meets several conditions together | `JsonTreeElementsQuery` | -| Jsonpath predicate | `predicate(text)` | Rows where a SQL/JSON path predicate is true | `JsonTreePredicateQuery` | -| Children and child count | `children(path)` | The entries directly below one node, or how many there are | A node token | +| Jsonpath predicate | `predicate(text)`, with `.vars(object)` | Rows where a SQL/JSON path predicate is true | `JsonTreePredicateQuery` | +| Children and child count | `children(path)` | The entries directly below one node, or how many there are | `JsonTreeToken` | A path is text. So an order query on a path the options do not order is refused at run time, before any key is fetched. @@ -525,13 +527,20 @@ page](../../packages/eql/docs/reference/json-tree.md) gives the SQL for each query. A scope chains on a JSON query as on a scalar one: `.contains(doc).scope(tenant)`. -The engine already has one private input type for each JSON operator, -in `stack_encrypt::dynamic`. `DocumentInput` is for containment, `AtInput` -for a comparison at a path, and `ElementsInput` for element match. The marker -methods build those same inputs, so the typed form and the dynamic form -share one set of input rules. The operand types `Contains`, `AtPath`, +The marker methods build the same inputs the dynamic interface reads a +binding's frames into (`json::inputs`, compiled without the `dynamic` +feature), so the typed form and the dynamic form share one set of input +rules and derive the same operands. The operand types `Contains`, `AtPath`, `ElementMatch`, `Predicate`, and `Children` that the plan proposed earlier -are retired, replaced by these methods, on 7 November 2026. +were retired as operands a caller builds, on 7 November 2026: a caller +writes the methods above. The query a method returns, +`marker::json::MarkedJson`, names its kind as a type with no +values, `Op`: `op::Contains`, `op::ContainedBy`, or `op::AtPath` of a +comparison, which the dynamic `Op` enum is generated from, and, for the +kinds that have no operator type, `marker::json::Selector`, +`ElementMatch`, `Predicate` and `Children`. That kind decides what +`encrypt` returns. Recorded on 16 November 2026, in #1459's pull +request. A document encrypted straight into `JsonTreeDocument`, with no record, has no marker. It is queried through a [query plan](#a-query-plan), as any @@ -1561,19 +1570,27 @@ Base64. returns the EQL query type. The marker supplies the column and the options. The dynamic parser keeps reading into the same input types, so there is one definition of each input. -- **Names.** The method names above are proposals. The operand types - `Contains`, `AtPath`, `ElementMatch`, `Predicate`, and `Children` that - this plan proposed earlier are retired, on 7 November 2026. +- **Names.** The method names above were confirmed in the naming review + of 8 November 2026. The operand types `Contains`, `AtPath`, + `ElementMatch`, `Predicate`, and `Children` that this plan proposed + earlier were retired as operands a caller builds, on 7 November 2026; + the query a method returns names its kind with value-free type tags of + those names (see [Query a JSON document](#query-a-json-document)). - **Why.** On `next`, a Rust program needs a plan for any JSON query, and needs the dynamic interface for element match, predicates, and children. Go already has typed methods for these. -- **Done already.** The input types and their parsing rules, as private - types in `stack_encrypt::dynamic::json_query`. The EQL query types - exist in `eql-bindings`. Typed containment and comparison at a path - went through the chain (`op::Contains`, `op::AtPath`), which #1666 - removes, so until this item is built every JSON query from Rust goes - through a query plan. -- **Tracked in.** Issue #1459 is open. It is exactly this item. Issue #1677 is open for opening one child. Issue #1486 is closed. +- **Done** in the pull request for #1459. Each method builds the input + the dynamic parser reads into (`json::inputs`, out of the `dynamic` + feature), and returns the field type's query type (`JsonQueries`): + the EQL types for `JsonTreeDocument`. A field's options are its + `#[stash(json(ordered = [..]))]`, decided on 16 November 2026. A test + holds every query to the data plan's operand, and the [JSON query + example](#query-a-json-document) runs. +- **Not done.** Typed paths checked at compile time, which #1459 asks + for: a path is text, checked before any key is fetched. Values are + typed (a number or a string for an ordering), and an element predicate + needs a condition to compile. +- **Tracked in.** Issue #1459 is open until its pull request merges. Issue #1677 is open for opening one child. Issue #1486 is closed. - **Finished when** these are both true: - the [JSON query example](#query-a-json-document) compiles; - an order query on a path the options do not order is refused before @@ -1605,8 +1622,7 @@ review. indexes, a nullable field and a record field their operator methods, and a field its query type (`#[stash(query = ..)]`). The chain query `cipher.query(op, value).on(&plan)` is removed (#1662). It was the only - typed JSON query, so until the JSON marker methods exist (#1459) a JSON - query goes through a query plan. + typed JSON query; a document field's marker methods replace it (#1459). - **No hand-built plan.** The review of 7 November 2026 gave a hand-built plan the same markers, through `Plan::::scope(..)` with `pick!`. That is withdrawn. No Rust program builds a plan by hand. diff --git a/languages/golang/encrypt/guest/src/status.rs b/languages/golang/encrypt/guest/src/status.rs index d7a829ae08..a48c8a2baa 100644 --- a/languages/golang/encrypt/guest/src/status.rs +++ b/languages/golang/encrypt/guest/src/status.rs @@ -155,6 +155,10 @@ pub fn status_for_error(error: &stack_encrypt::Error) -> u32 { // children asked for as a scalar, or an altered entry. The client // cannot tell which, so neither STATUS_AUTH nor a claim of either. stack_encrypt::Error::JsonNotALeaf => STATUS_ENCODING, + // A refusal of the caller's input, reported through the crate's + // error by a path whose failure can only be that: the dynamic + // refusal it holds is classified as the dynamic path classifies it. + stack_encrypt::Error::Input(refusal) => status_for_dynamic(refusal), _ => STATUS_INTERNAL, } } @@ -422,6 +426,21 @@ mod tests { ); } + /// A refusal of the caller's input that arrives in the crate's error is + /// classified as the dynamic path classifies it. + #[test] + fn an_input_refusal_is_the_callers_input() { + use stack_encrypt::refusal::{Error, Reason}; + let refused = Error::Source { + field: None, + reason: Reason::JsonPathNoWildcard, + }; + assert_eq!( + status_for_error(&stack_encrypt::Error::Input(Box::new(refused))), + STATUS_ENCODING + ); + } + #[test] fn a_context_mismatch_is_its_own_status() { use stack_encrypt::Describe; diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index e60b39ee13..0862fcd254 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -9,6 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **A `JsonTreeDocument` field is queried from its marker** (#1459). As a + field of a derived record, a `JsonTreeDocument` (and an + `Option`) has Stack Encrypt's JSON marker queries, + which return the EQL query types a parameter binds as: + `EncryptedCustomer::PROFILE.contains(doc)` gives a `JsonTreeQuery`, + `at(path).gt(v)` a `JsonTreeLeafQuery`, `elements(path)..` a + `JsonTreeElementsQuery`, `predicate(text)` a `JsonTreePredicateQuery`, + `at(path).selector()` a `JsonTreeSelector` and `children(path)` a + `JsonTreeToken`. A field marked `#[stash(json(ordered = ["$.age"]))]` is + sealed under those options, so its orderings compare. Each operand is + the one a data plan naming `JsonSearch` derives for the same column and + options. `JsonTreeDocument` implements `JsonTarget` and `JsonQueries`. + - **The parts of a `JsonSearch` document decrypt through the owned API** (#1660). `JsonTreeExtracted`, the entry `->` returns, implements `DecryptSource`. `JsonTreeSubtree`, the rows of a subtree @@ -333,8 +346,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 gone. - A JSON document is a `JsonPlaintext`, not a `serde_json::Value`. `JsonTreeDocument`, `JsonTreeExtracted` and `JsonTreeSubtree` open to - it. A typed JSON query has no replacement yet: derive the operand - through a data plan that names `JsonSearch` (`EqlTargets`). + it. A typed JSON query is a document field's marker query (above). - `JsonTree` no longer implements `stack_encrypt::target::Layout`, which is removed. `JsonTree::encryption()`, `index()` and `specs()` are its own methods, and `index()` returns the index by value. diff --git a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs index 36542fd979..4c4dcb4c71 100644 --- a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs +++ b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs @@ -12,7 +12,13 @@ //! [`EncryptTarget`] / [`DecryptSource`] impls that let a Rust program name //! [`JsonTreeDocument`] as a target: //! `document.encrypt_into_with_context(&keyset, column)` under an -//! [`Identifier`]. A query derives through a data plan that names the type +//! [`Identifier`]. As a field of a derived record, its marker has the +//! JSON queries (`contains`, `at`, `elements`, `predicate`, `children`), +//! which return the query types here +//! ([`JsonQueries`](stack_encrypt::json::JsonQueries)), under the options +//! the field's `#[stash(json(..))]` names +//! ([`JsonTarget`](stack_encrypt::json::JsonTarget)). A document with +//! no record is queried through a data plan that names the type //! (`JsonSearch`) under the same column, and its operand converts into the //! operand types here (see [`crate::encryption`]). The //! design is `docs/plans/2026-10-17-json-tree-index.md` at @@ -22,8 +28,9 @@ use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; use serde_json::Value; use stack_encrypt::json::{ - JsonDocument, JsonLeafQuery, JsonNode, JsonOptions, JsonPath, JsonPlaintext, JsonQuery, - JsonSelector, JsonSubtree, JsonSubtreeKey, + JsonDocument, JsonElementsQuery, JsonLeafQuery, JsonNode, JsonOptions, JsonPath, JsonPlaintext, + JsonPredicateQuery, JsonQueries, JsonQuery, JsonSelector, JsonSubtree, JsonSubtreeKey, + JsonTarget, }; use stack_encrypt::target::{ json_document, AeadContext, Borrowed, DecryptSource, EncryptTarget, ExpectedContext, Index, @@ -559,6 +566,59 @@ impl EncryptTarget for JsonTreeDocument { } } +/// A document field sealed under the options its record's +/// `#[stash(json(..))]` names, which order the paths its queries compare. +impl JsonTarget for JsonTreeDocument { + fn json_encryption<'s, K: 'static>( + options: JsonOptions, + ) -> Encryption<'s, JsonPlaintext, Self, K, Self::ContextInput> + where + JsonPlaintext: 's, + { + JsonTree::new(options) + .sealing() + .project(JsonPlaintext::value) + } +} + +/// A document field's marker queries return the EQL query types a +/// parameter binds as: `JsonTreeQuery` for `@>` and `<@` +/// (`eql_v4.query_json`), `JsonTreeLeafQuery` for an ordering at a path +/// (`eql_v4.query_json_leaf`), `JsonTreeElementsQuery` for +/// `eql_v4.json_tree_element_match`, `JsonTreePredicateQuery` for `@@`, the +/// selector `->` takes, and the token the child functions take. +impl JsonQueries for JsonTreeDocument { + type Query = JsonTreeQuery; + type LeafQuery = JsonTreeLeafQuery; + type ElementsQuery = JsonTreeElementsQuery; + type PredicateQuery = JsonTreePredicateQuery; + type Selector = JsonTreeSelector; + type Token = JsonTreeToken; + fn query(query: JsonQuery) -> Result { + JsonTreeQuery::try_from(query) + } + fn leaf_query(query: JsonLeafQuery) -> Result { + JsonTreeLeafQuery::try_from(&query) + } + fn elements_query(query: JsonElementsQuery) -> Result { + transcode(&query) + } + fn predicate_query(query: JsonPredicateQuery) -> Result { + transcode(&query) + } + fn selector(selector: JsonSelector) -> Result { + transcode(&selector) + } + fn token(key: JsonSubtreeKey) -> Result { + match key.tokens().as_slice() { + [token] => transcode(token), + _ => Err(Error::Other( + "a path with no wildcard names one node".into(), + )), + } + } +} + /// Decrypts to the plaintext document, under the stored column. /// /// - `decrypt_from_with_context(document, &keyset, column)` checks that the diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index e1d7133b58..55811bc250 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -4,8 +4,9 @@ //! (`JsonTreeDocument`, `JsonTreeExtracted`, `JsonTreeSubtree`), as a //! `JsonPlaintext`. A query's operand derives through a data plan that //! names the type `JsonSearch` under the same column and options -//! ([`contains`], [`contained_by`], [`gte_at`]), since an EQL type has no -//! typed JSON query. The unit tests need no database; the PostgreSQL test +//! ([`contains`], [`contained_by`], [`gte_at`]), the path a language SDK +//! takes. A record field's marker derives the same operands +//! (`tests/json_tree_marker.rs`). The unit tests need no database; the PostgreSQL test //! installs nothing itself (`mise run test:encryption:postgres` installs the //! v4 bundle built from the tree) and is the oracle the design asks for: //! `@>` and `<@` over the encrypted column against PostgreSQL's own @@ -61,9 +62,9 @@ const ORDERED: [&str; 2] = ["$.hello", "$.number"]; /// The plan a query derives through: a data plan with one `json` field, /// `profile`, that names the EQL type `JsonSearch` under `users/profile`. /// That context is the identifier [`seal`] stores (the same two parts), so -/// a query's selectors are the stored document's. An EQL type has no typed -/// JSON query: a Rust program derives one through a data plan, as a -/// language SDK does. +/// a query's selectors are the stored document's. A document sealed on its +/// own, with no record, has no marker, so it is queried through a data +/// plan, as a language SDK queries one. fn plan() -> record::Plan { let wire = json!({ "profile": { diff --git a/packages/eql/tests/encryption/tests/json_tree_marker.rs b/packages/eql/tests/encryption/tests/json_tree_marker.rs new file mode 100644 index 0000000000..df311f69f3 --- /dev/null +++ b/packages/eql/tests/encryption/tests/json_tree_marker.rs @@ -0,0 +1,338 @@ +//! An encrypted JSON document field queried from its marker: the typed +//! Rust form of the JSON operators, returning the EQL query types a +//! parameter binds as. +//! +//! The record seals the field under the options its `#[stash(json(..))]` +//! names, and the field's marker derives each query under the same column +//! and options. Each query is held to the operand a data plan of the same +//! declaration derives, which is the path the Go SDK takes and the one the +//! PostgreSQL suite checks against `jsonb`, and to the document the record +//! sealed: +//! +//! - `contains`, `contained_by` and `at(path).eq` derive the containment +//! needle (`JsonTreeQuery`), and find the documents that hold it; +//! - the orderings at a path derive `JsonTreeLeafQuery`, with the path's +//! selector, and `between` the two bounds; +//! - `elements(path)` derives `JsonTreeElementsQuery`, `predicate(text)` +//! `JsonTreePredicateQuery`, and `children(path)` the node's token; +//! - an ordering at a path the field does not order is refused before any +//! key is requested. + +mod common; + +use std::num::NonZeroU32; + +use eql_bindings::encryption::EqlTargets; +use eql_bindings::v4::json_tree::{ + JsonTreeDocument, JsonTreeElementsQuery, JsonTreeLeafQuery, JsonTreePredicateQuery, + JsonTreeQuery, JsonTreeSelector, JsonTreeToken, +}; +use serde_json::{json, Value}; +use stack_encrypt::dynamic::{self, record, ValueFrame}; +use stack_encrypt::json::{JsonDocument, JsonPlaintext, JsonQuery}; +use stack_encrypt::sem::TermError; +use stack_encrypt::target::{DecryptSource, EncryptTarget}; +use stack_encrypt::{AsProtectedRef as _, DecryptFrom, EncryptInto, Error, KeysetCipher}; +use zeroize::Zeroize; + +/// A customer, and the profile document stored for them. +#[derive(Zeroize)] +struct Customer { + profile: JsonPlaintext, +} + +/// The plan's example: the profile is stored under `customers/profile`, +/// ordering the age and each visit's time and place. +#[derive(EncryptTarget, DecryptSource)] +#[stash(struct = Customer, scope = "customers")] +struct EncryptedCustomer { + #[stash(json(ordered = ["$.age", "$.visits[*].at", "$.visits[*].place"]))] + profile: JsonTreeDocument, +} + +fn profile() -> Value { + json!({ + "name": "Ada", + "age": 35, + "visits": [{"place": "home", "at": 15}, {"place": "work", "at": 9}] + }) +} + +/// The field as a data plan, under the record's options. +fn plan() -> record::Plan { + let wire = json!({ + "profile": { + "context": ["customers", "profile"], + "type": "json", + "null": "sql", + "target": { + "name": "JsonSearch", + "options": {"ordered": ["$.age", "$.visits[*].at", "$.visits[*].place"]} + } + } + }); + record::plan_with( + dynamic::frame::serde::to_vec(&wire) + .unwrap() + .as_protected_ref(), + &EqlTargets, + ) + .unwrap() +} + +/// The data plan's operand for `operator` and its input. +async fn data( + keyset: &KeysetCipher<'_, common::Observed>, + operator: &str, + input: Value, +) -> dynamic::Operand { + let frame = ValueFrame::from_bytes( + dynamic::frame::serde::to_vec(&input) + .unwrap() + .as_protected_ref(), + ) + .unwrap(); + dynamic::query_term( + keyset, + &plan(), + "profile", + dynamic::Op::parse(operator).unwrap(), + frame, + &EqlTargets, + ) + .unwrap() + .await + .unwrap() + .operand() + .clone() +} + +fn one(operand: dynamic::Operand) -> Value { + match operand { + dynamic::Operand::One(bytes) => serde_json::from_slice(&bytes).unwrap(), + other => panic!("one operand: {other:?}"), + } +} + +fn text(operand: dynamic::Operand) -> String { + match operand { + dynamic::Operand::Text(text) => text, + other => panic!("text: {other:?}"), + } +} + +/// A containment needle as the engine's `JsonQuery`, which compares as +/// sets: each derivation sends its selector set in a random order of its +/// own. +fn needle(query: &JsonTreeQuery) -> JsonQuery { + serde_json::from_value(serde_json::to_value(query).unwrap()).unwrap() +} + +/// The JSON index's own document under the EQL envelope, whose selector +/// set compares as a set. +fn index_document(document: &JsonTreeDocument) -> JsonDocument { + let wire = serde_json::to_value(document).unwrap(); + serde_json::from_value(json!({"h": wire["h"], "sv": wire["sv"], "q": wire["q"]})).unwrap() +} + +/// The plan's "Query a JSON document" example, run: each query's type is +/// the EQL query type the marker fixes. +#[tokio::test] +async fn the_plans_json_query_example_runs() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let stored: EncryptedCustomer = Customer { + profile: profile().into(), + } + .encrypt_into(&keyset) + .await + .unwrap(); + + let ada: JsonTreeQuery = EncryptedCustomer::PROFILE + .contains(json!({ "name": "Ada" })) + .encrypt(&keyset) + .await + .unwrap(); + let older: JsonTreeLeafQuery = EncryptedCustomer::PROFILE + .at("$.age") + .gt(30) + .encrypt(&keyset) + .await + .unwrap(); + let (low, high): (JsonTreeLeafQuery, JsonTreeLeafQuery) = EncryptedCustomer::PROFILE + .at("$.age") + .between(18, 65) + .encrypt(&keyset) + .await + .unwrap(); + let home: JsonTreeElementsQuery = EncryptedCustomer::PROFILE + .elements("$.visits[*]") + .eq("place", "home") + .gt("at", 10) + .encrypt(&keyset) + .await + .unwrap(); + let adult: JsonTreePredicateQuery = EncryptedCustomer::PROFILE + .predicate("$.age >= $min") + .vars(json!({"min": 18})) + .encrypt(&keyset) + .await + .unwrap(); + let visits: JsonTreeToken = EncryptedCustomer::PROFILE + .children("$.visits") + .encrypt(&keyset) + .await + .unwrap(); + let age: JsonTreeSelector = EncryptedCustomer::PROFILE + .at("$.age") + .selector() + .encrypt(&keyset) + .await + .unwrap(); + + assert_eq!( + older.s, age, + "an ordering is compared at the path's selector" + ); + assert_eq!((&low.s, &high.s), (&age, &age)); + assert_eq!(home.m.len(), 2, "one member per condition"); + assert!(matches!(adult, JsonTreePredicateQuery::And(_)), "{adult:?}"); + assert!( + serde_json::to_value(&visits).unwrap().is_string(), + "a token is text" + ); + assert!( + needle(&ada).matches(index_document(&stored.profile).selector_set()), + "the stored document holds the needle" + ); + + let opened = Customer::decrypt_from(stored, &keyset).await.unwrap(); + assert_eq!(opened.profile.value(), &profile()); +} + +/// Every marker query derives the operand the data plan derives for the +/// same declaration. +#[tokio::test] +async fn each_query_is_the_data_plans_operand() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + + let typed = EncryptedCustomer::PROFILE + .contains(json!({"visits": [{"place": "home"}]})) + .encrypt(&keyset) + .await + .unwrap(); + let wire = one(data( + &keyset, + "contains", + json!({"json": json!({"visits": [{"place": "home"}]}).to_string()}), + ) + .await); + assert_eq!( + needle(&typed), + serde_json::from_value::(wire).unwrap() + ); + + let typed = EncryptedCustomer::PROFILE + .at("$.visits[*].place") + .eq("home") + .encrypt(&keyset) + .await + .unwrap(); + let wire = one(data( + &keyset, + "eq_at", + json!({"path": "$.visits[*].place", "json": "\"home\""}), + ) + .await); + assert_eq!( + needle(&typed), + serde_json::from_value::(wire).unwrap() + ); + + let typed = EncryptedCustomer::PROFILE + .at("$.visits[*].at") + .lte(20) + .encrypt(&keyset) + .await + .unwrap(); + let wire = one(data( + &keyset, + "lte_at", + json!({"path": "$.visits[*].at", "json": "20"}), + ) + .await); + assert_eq!(serde_json::to_value(&typed).unwrap(), wire); + + let typed = EncryptedCustomer::PROFILE + .elements("$.visits[*]") + .eq("place", "home") + .between("at", 10, 20) + .wildcard_bound(NonZeroU32::new(8).unwrap()) + .encrypt(&keyset) + .await + .unwrap(); + let wire = one(data( + &keyset, + "element_match", + json!({"path": "$.visits[*]", "wildcard_bound": 8, "predicates": [ + {"member": "place", "op": "eq", "json": "\"home\""}, + {"member": "at", "op": "between", "low": "10", "high": "20"} + ]}), + ) + .await); + assert_eq!(serde_json::to_value(&typed).unwrap(), wire); + + let typed = EncryptedCustomer::PROFILE + .predicate("$.age >= $min && exists($.name)") + .vars(json!({"min": 30})) + .encrypt(&keyset) + .await + .unwrap(); + let wire = one(data( + &keyset, + "match", + json!({"predicate": "$.age >= $min && exists($.name)", "vars": "{\"min\": 30}"}), + ) + .await); + assert_eq!(serde_json::to_value(&typed).unwrap(), wire); + + let typed = EncryptedCustomer::PROFILE + .at("$.age") + .selector() + .encrypt(&keyset) + .await + .unwrap(); + let wire = text(data(&keyset, "selector", json!({"path": "$.age"})).await); + assert_eq!(serde_json::to_value(&typed).unwrap(), json!(wire)); + + let typed = EncryptedCustomer::PROFILE + .children("$.visits") + .encrypt(&keyset) + .await + .unwrap(); + let wire = text(data(&keyset, "children", json!({"path": "$.visits"})).await); + assert_eq!(serde_json::to_value(&typed).unwrap(), json!(wire)); +} + +/// An ordering at a path the field does not order is refused before any +/// key is requested, as the data plan refuses it. +#[tokio::test] +async fn an_ordering_the_field_does_not_order_is_refused_before_any_key() { + let (cipher, calls) = common::cipher().await; + let keyset = cipher.default_keyset(); + let refused = EncryptedCustomer::PROFILE + .at("$.name") + .gt("A") + .encrypt(&keyset) + .await; + assert!( + matches!(refused, Err(Error::Term(TermError::JsonPathNotOrdered))), + "{refused:?}" + ); + let calls = calls.lock().unwrap(); + assert!( + calls.generate.is_empty() && calls.retrieve.is_empty(), + "no key requested" + ); +} diff --git a/packages/eql/tests/encryption/tests/typed_queries.rs b/packages/eql/tests/encryption/tests/typed_queries.rs index 7dbb2e026d..794c2f9fdb 100644 --- a/packages/eql/tests/encryption/tests/typed_queries.rs +++ b/packages/eql/tests/encryption/tests/typed_queries.rs @@ -213,8 +213,9 @@ async fn typed_and_input(keyset: &Keyset<'_>, name: &str, operator: &str) -> Typ #[test] fn a_marker_answers_what_the_data_path_answers() { let mut gap = Vec::new(); - // A document has no marker query: it is queried through a data plan, - // which tests/json_tree_data_plan.rs covers. + // A document's marker answers through its JSON queries + // (`JsonQueries`), not `Answers`: tests/json_tree_marker.rs holds them + // to the data path. for target in eql_targets::targets() .iter() .filter(|t| t.plaintext != Some("json")) diff --git a/packages/stack-encrypt-derive/docs/attributes.md b/packages/stack-encrypt-derive/docs/attributes.md index 196395122a..446828d2b8 100644 --- a/packages/stack-encrypt-derive/docs/attributes.md +++ b/packages/stack-encrypt-derive/docs/attributes.md @@ -42,6 +42,7 @@ field from the whole `User`. To encrypt a struct field by field, use | `default` or `default = expr` | The field is not derived. It is filled with `Default::default()` or `expr`. It is not encrypted and not authenticated. | | `decrypt` | Decryption opens this field. Use it only when the field types do not decide it. See [Which field decryption opens](#which-field-decryption-opens). | | `passthrough` | Owned derives, with `struct` only. The plaintext field is moved into the record as it is, and moved back on decryption. It is **not encrypted and not authenticated**. See [Passthrough fields](#passthrough-fields). Not with `identity`, `default` or `decrypt`. | +| `json(ordered = ["$.path", ..], sealed_root = false)` | Owned derives, with `struct` only, on a field whose type stores a JSON document (`JsonDocument`, or EQL's `JsonTreeDocument`). The field is sealed under the JSON index options it names: `ordered` lists the paths whose numbers and strings carry an ordering term, and `sealed_root = false` turns off the sealed root, which is on by default. The field's marker queries under the same options, so `PROFILE.at("$.age").gt(30)` compares at a path the field orders. A path is JSONPath text, checked when the field is sealed or queried, before any key is requested. Without it, the field takes the default options, which order no path. | A field cannot have a `context = ".."` of its own, on any derive. Every field of a `plaintext` record shares the caller's context. Every field of a diff --git a/packages/stack-encrypt-derive/src/attrs.rs b/packages/stack-encrypt-derive/src/attrs.rs index 246f8b2272..93526a04f0 100644 --- a/packages/stack-encrypt-derive/src/attrs.rs +++ b/packages/stack-encrypt-derive/src/attrs.rs @@ -1,6 +1,7 @@ //! Parsing of the `#[stash(...)]` container and field attributes. -use syn::{Attribute, Expr, Ident, LitStr, Member, Path, Result, Type}; +use syn::punctuated::Punctuated; +use syn::{Attribute, Expr, Ident, LitBool, LitStr, Member, Path, Result, Token, Type}; /// Container-level options, from `#[stash(...)]` on the struct itself. pub(crate) struct ContainerAttrs { @@ -342,6 +343,62 @@ pub(crate) struct FieldAttrs { /// an `Option` of it. On a field whose marker has no operator method /// (a record field, a ciphertext alone), only the check applies. pub(crate) query: Option, + /// `#[stash(json(ordered = ["$.age"], sealed_root = false))]`: with + /// `struct = ..`, the JSON index's options of a field that stores a + /// document. The field is sealed under them and its marker's JSON + /// queries derive under them. + pub(crate) json: Option, +} + +/// The JSON index's options a field's `json(..)` names. +#[cfg_attr(test, derive(Debug))] +pub(crate) struct JsonAttr { + /// Where `json` is written, for errors. + pub(crate) span: proc_macro2::Span, + /// `ordered = ["$.age", ..]`: the paths whose leaves carry an ordering + /// term. JSONPath text, parsed by the engine when the field is sealed. + pub(crate) ordered: Vec, + /// `sealed_root = false`: turn off the sealed root, which is on by + /// default. + pub(crate) sealed_root: Option, +} + +impl JsonAttr { + fn parse(meta: &syn::meta::ParseNestedMeta<'_>) -> Result { + let mut json = Self { + span: syn::spanned::Spanned::span(&meta.path), + ordered: Vec::new(), + sealed_root: None, + }; + let mut ordered = false; + meta.parse_nested_meta(|inner| { + if inner.path.is_ident("ordered") { + if ordered { + return Err(inner.error("`ordered` is given twice")); + } + ordered = true; + let value = inner.value()?; + let content; + syn::bracketed!(content in value); + json.ordered = Punctuated::::parse_terminated(&content)? + .into_iter() + .collect(); + return Ok(()); + } + if inner.path.is_ident("sealed_root") { + if json.sealed_root.is_some() { + return Err(inner.error("`sealed_root` is given twice")); + } + json.sealed_root = Some(inner.value()?.parse()?); + return Ok(()); + } + Err(inner.error( + "unsupported `json(..)` option; expected `ordered = [\"$.path\", ..]` or \ + `sealed_root = false`", + )) + })?; + Ok(json) + } } impl FieldAttrs { @@ -421,6 +478,15 @@ impl FieldAttrs { parsed.passthrough = true; return Ok(()); } + if meta.path.is_ident("json") { + if parsed.json.is_some() { + return Err(meta.error( + "`json(..)` is given twice; a field's JSON options are one list", + )); + } + parsed.json = Some(JsonAttr::parse(&meta)?); + return Ok(()); + } if meta.path.is_ident("query") { if parsed.query.is_some() { return Err( @@ -445,7 +511,7 @@ impl FieldAttrs { Err(meta.error( "unsupported field attribute; expected `context_field`, \ `identity = \"...\"`, `from = field`, `default`, `default = expr`, \ - `decrypt`, `passthrough` or `query = Type`", + `decrypt`, `passthrough`, `query = Type` or `json(..)`", )) })?; } diff --git a/packages/stack-encrypt-derive/src/lib.rs b/packages/stack-encrypt-derive/src/lib.rs index 53142a7fe0..000e34f71f 100644 --- a/packages/stack-encrypt-derive/src/lib.rs +++ b/packages/stack-encrypt-derive/src/lib.rs @@ -135,6 +135,14 @@ //! inner record lists a scope, the query needs each record's scope, //! outermost first. //! +//! `#[stash(json(ordered = ["$.age"]))]` on a field whose type stores a +//! JSON document names the JSON index's options: the paths whose numbers +//! and strings carry an ordering term, and `sealed_root = false` to turn +//! the sealed root off. The field is sealed under them, and its marker's +//! JSON queries (`contains`, `at`, `elements`, `predicate`, `children`) +//! derive under them. Without it the field takes the default options, +//! which order no path. +//! //! `#[stash(query = String)]` on a field states the type its queries take, //! so each of its marker's methods takes that type alone. The derive checks //! it against the plaintext struct's field, which is that type or an diff --git a/packages/stack-encrypt-derive/src/owned_encrypt.rs b/packages/stack-encrypt-derive/src/owned_encrypt.rs index 1062d9fe51..ac3e600330 100644 --- a/packages/stack-encrypt-derive/src/owned_encrypt.rs +++ b/packages/stack-encrypt-derive/src/owned_encrypt.rs @@ -31,7 +31,66 @@ pub(crate) fn derive(input: DeriveInput) -> Result { }; let contexts = contexts_fn(&input, &record); let markers = markers(&input, &record); - Ok(quote!(#emitted #decryptable #contexts #markers)) + let json = json_options_fns(&input, &record); + Ok(quote!(#emitted #decryptable #contexts #markers #json)) +} + +/// The name of the function that builds the JSON options a field's +/// `#[stash(json(..))]` names, which the field's encryption and its marker +/// both call. +fn json_options_name(field: &Field) -> Ident { + Ident::new( + &format!("__stash_json_options_{}", field.local), + field.local.span(), + ) +} + +/// One function per `#[stash(json(..))]` field, building the options it +/// names: the paths it orders, and whether the root is sealed. A path the +/// engine refuses is an error the field's encryption and its queries meet +/// before any key is requested. +fn json_options_fns(input: &DeriveInput, record: &Record) -> TokenStream { + let krate = &record.krate; + let name = &input.ident; + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + let fns: Vec = record + .fields + .iter() + .filter_map(|field| { + let Kind::Derived { + by_field: Some(ByField { json: Some(json), .. }), + } = &field.kind + else { + return None; + }; + let fn_name = json_options_name(field); + let paths = &json.ordered; + let count = paths.len(); + let sealed_root = json.sealed_root.as_ref().map(|sealed| { + quote!(options.sealed_root = #sealed;) + }); + Some(quote_spanned! {json.span=> + #[doc(hidden)] + #[allow(dead_code)] + fn #fn_name() -> ::core::result::Result<#krate::json::JsonOptions, #krate::sem::TermError> { + let paths: [&'static str; #count] = [#(#paths),*]; + #[allow(unused_mut)] + let mut options = #krate::json::JsonOptions::new().ordering(paths)?; + #sealed_root + ::core::result::Result::Ok(options) + } + }) + }) + .collect(); + if fns.is_empty() { + return TokenStream::new(); + } + quote! { + #[automatically_derived] + impl #impl_generics #name #ty_generics #where_clause { + #(#fns)* + } + } } /// The field markers of a `struct = ..` record: one associated constant @@ -61,11 +120,15 @@ fn markers(input: &DeriveInput, record: &Record) -> TokenStream { }; let consts = record.fields.iter().filter_map(|field| { let Kind::Derived { - by_field: Some(ByField { context, query, .. }), + by_field: Some(ByField { context, query, json, .. }), } = &field.kind else { return None; }; + let json_options = json.as_ref().map(|_| { + let fn_name = json_options_name(field); + quote!(.with_json_options(Self::#fn_name)) + }); let syn::Member::Named(ident) = &field.member else { return None; }; @@ -91,7 +154,7 @@ fn markers(input: &DeriveInput, record: &Record) -> TokenStream { #[doc = #doc] #[allow(dead_code, private_interfaces)] pub const #const_name: #krate::Marker<#krate::marker::Root, #ty, #scope_ty, #query_ty> = - #krate::Marker::new(#field_name, #prefix, #identity, Self::contexts); + #krate::Marker::new(#field_name, #prefix, #identity, Self::contexts)#json_options; }) }); quote! { @@ -320,7 +383,7 @@ fn struct_derive(input: &DeriveInput, record: &Record) -> TokenStream { let local = &field.local; match &field.kind { Kind::Derived { - by_field: Some(ByField { from, context, query }), + by_field: Some(ByField { from, context, query, json }), } => { let own = own_expr(krate, context); let name = field_name(field); @@ -334,9 +397,18 @@ fn struct_derive(input: &DeriveInput, record: &Record) -> TokenStream { }; } }); + // A `json(..)` field is sealed under the options it + // names, through the field type's `JsonTarget`. + let description = match json { + Some(_) => { + let fn_name = json_options_name(field); + quote_spanned!(ty.span()=> #krate::json::seal_field::<#ty, _, __K>(Self::#fn_name())) + } + None => quote_spanned!(ty.span()=> <#ty as #krate::target::EncryptTarget<_>>::encryption::<__K>()), + }; quote_spanned! {ty.span()=> #check - let #local = <#ty as #krate::target::EncryptTarget<_>>::encryption::<__K>() + let #local = #description .under_field(#own) .project(|__source: &#source| &__source.#from) .named(#name); @@ -367,14 +439,21 @@ fn struct_derive(input: &DeriveInput, record: &Record) -> TokenStream { let local = &field.local; match &field.kind { Kind::Derived { - by_field: Some(ByField { context, .. }), + by_field: Some(ByField { context, json, .. }), } => { let own = own_expr(krate, context); let name = field_name(field); + let description = match json { + Some(_) => { + let fn_name = json_options_name(field); + quote_spanned!(ty.span()=> #krate::json::seal_field_owned::<#ty, _, __K>(Self::#fn_name())) + } + None => quote_spanned!(ty.span()=> <#ty as #krate::target::EncryptTarget<_>>::owned_encryption::<__K>()), + }; // `under_field` wipes the field's plaintext when the // context is refused before the field's target runs. quote_spanned! {ty.span()=> - let #local = <#ty as #krate::target::EncryptTarget<_>>::owned_encryption::<__K>() + let #local = #description .under_field(#own) .named(#name); } diff --git a/packages/stack-encrypt-derive/src/shape.rs b/packages/stack-encrypt-derive/src/shape.rs index 8fb05e2dad..9bd6d578d7 100644 --- a/packages/stack-encrypt-derive/src/shape.rs +++ b/packages/stack-encrypt-derive/src/shape.rs @@ -8,7 +8,7 @@ use syn::{ PathArguments, Result, Type, }; -use crate::attrs::{ContainerAttrs, FieldAttrs}; +use crate::attrs::{ContainerAttrs, FieldAttrs, JsonAttr}; /// A generated lifetime must not shadow one the record declares. Append /// underscores until the name is free, preserving the user's parameters. @@ -76,6 +76,9 @@ pub(crate) struct ByField { /// `#[stash(query = T)]`: the type a query of the field takes, for its /// marker. pub(crate) query: Option, + /// `#[stash(json(..))]`: the JSON index's options the field is sealed + /// and queried under. + pub(crate) json: Option, } impl Field { @@ -473,6 +476,23 @@ fn collect(fields: &Fields, prefix: Option<&LitStr>) -> Result> { )); } } + // JSON options are a sealed document field's, and only a field + // of a `struct = ..` record has a marker to query it with. + if let Some(json) = &attrs.json { + if prefix.is_none() + || attrs.context_field + || attrs.passthrough + || attrs.default.is_some() + { + return Err(syn::Error::new( + json.span, + "`json(..)` names the JSON index's options a sealed document field is \ + sealed and queried under; only a sealed field of a `struct = ..` record \ + has them, so it does not apply to a `plaintext` record's field, or to a \ + `context_field`, `passthrough` or `default` field", + )); + } + } // An identity is one label segment: non-empty, and plain so the // descriptor names the field. if let Some(identity) = &attrs.identity { @@ -588,6 +608,7 @@ fn collect(fields: &Fields, prefix: Option<&LitStr>) -> Result> { from, context, query: attrs.query, + json: attrs.json, }), } } diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 2a260bdc82..976f94ad7b 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -9,6 +9,69 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Typed JSON queries from a field marker** (#1459). A field whose type + stores a JSON document (`JsonDocument`, or EQL's `JsonTreeDocument`) has + one marker method for each kind of JSON query, in place of the operator + methods: + + | Marker method | Returns, for `JsonDocument` | + | --- | --- | + | `contains(doc)`, `contained_by(doc)`, `at(path).eq(v)` | `JsonQuery` | + | `at(path).lt(v)`, `.lte`, `.gt`, `.gte` | `JsonLeafQuery` | + | `at(path).between(lo, hi)` | `(JsonLeafQuery, JsonLeafQuery)`, low then high | + | `at(path).is_null()` | `JsonQuery` | + | `at(path).selector()` | `JsonSelector` | + | `elements(path).eq(member, v).gt(member, v)` | `JsonElementsQuery` | + | `predicate(text).vars(object)` | `JsonPredicateQuery` | + | `children(path)` | `JsonSubtreeKey`, which holds the node's one token | + + `EncryptedCustomer::PROFILE.at("$.age").gt(30).encrypt(&keyset)` derives + under the field's column and options, as a scalar field's query does, + and takes a scope with `.scope(..)`. Each query derives the operand the + dynamic interface derives for the same declaration: the marker and the + data plan read one definition of each input, which no longer needs the + `dynamic` feature. + - **Field options.** `#[stash(json(ordered = ["$.age", "$.visits[*].at"], + sealed_root = false))]` on the field names the JSON index's options. + The derive seals the field under them, and the marker queries under + them. Without it the field takes the default options, which order no + path, so every ordering at a path is refused. A document type that + seals under other options when its field names none says so with + `JsonQueries::default_options`, which its marker queries under. + - **Typed values.** An ordering takes a number or a string + (`marker::json::JsonOrdered`), equality at a path a scalar + (`JsonScalar`), and containment a document. A boolean ordering, or + equality with an object, does not compile. `elements` with no + condition has no `encrypt`. A NaN or an infinity, which no JSON number + is, is refused with the new `TermError::JsonNumberNotFinite` + (`stack_encrypt::json_number_not_finite`) before any key is requested, + rather than becoming `null`. An `f32` is the number its shortest + decimal form names, so `0.1f32` finds a stored `0.1`. + - **Refusals.** A path is JSONPath text, checked when the query is + encrypted, before any key is requested, by the dynamic interface's + rules: an ordering at a path the field does not order is + `TermError::JsonPathNotOrdered`. A refusal of the input's own shape (an + `elements` path with no wildcard, a predicate the grammar does not + read) is the new `Error::Input`, below. + - **New public items.** `json::JsonTarget`, which seals a document under + options; `json::JsonQueries`, which converts each operand into the + field type's query type; `json::JsonElementsQuery` and + `json::JsonPredicateQuery`, the typed element and predicate operands; + and `marker::json`, the query types. + +- **`Error::Input`: a refusal of the caller's input** (#1459). A field + marker's JSON query whose input has the wrong shape, and a data plan run + through `dynamic::Planned` whose source does not fit it, now fail with + `Error::Input`, which holds the dynamic interface's error. Its message, + code (`stack_encrypt::dynamic_source`, `stack_encrypt::dynamic_predicate` + and the rest) and payload (`reason`, and `field`, `construct` and `at` + where it has them) are that error's, so a refusal reports the same way + on both paths. They used to be `Error::Other`, whose code + (`stack_encrypt::other`) a caller could not tell from an engine fault. + The error type is now public without the `dynamic` feature too, as + `stack_encrypt::refusal::Error` and `refusal::Reason` (the same types as + `dynamic::Error` and `dynamic::Reason`). This module's own bug + (`Internal`) and a target resolver's own failure stay `Error::Other`. - **Two more records derive** (#1661). The owned derives did not accept these shapes, and the removed derives did: - A `plaintext = T` record can have a `struct = ..` record as a field. @@ -145,8 +208,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 them. The caller's scope and bindings are the one context argument, after the plan's own. A query takes scopes alone, and refuses a binding before any key is requested, as `PlanError::BindInQuery`. A refused - term keeps its kind (`Error::Term`); other dynamic refusals are - `Error::Other`. `Planned::contexts()` shows every field's context with + term keeps its kind (`Error::Term`); a refusal of the caller's input is + `Error::Input`. `Planned::contexts()` shows every field's context with a placeholder for the caller's scope and one for the caller's binding, before any value is encrypted. - **Query plans and the dynamic field handle** (#1486). `Plan::query(field, @@ -164,6 +227,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Breaking +- **An index run over a `ValueFrame` refuses a value as `Error::Input`** + (#1459). An `IndexSpec` run directly over a `ValueFrame` it has no term + for (a boolean under equality, a container under ORE) failed with the + dynamic `Error::Term` boxed in `Error::Other`. It is now in + `Error::Input`, matched as `Error::Input(refusal)` with + `refusal.as_ref()` the `dynamic::Error::Term`, and its code is + `stack_encrypt::dynamic_term` rather than `stack_encrypt::other`. - **The structural API is removed** (#1661, #1662). Stack Encrypt had two ways to encrypt a record and two ways to query one. The owned API stays: `EncryptTarget`, `DecryptSource`, `EncryptInto`, `DecryptFrom`, field @@ -201,12 +271,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 | `plan::all((a, b))` | `a.zip(b)` on two `Pending` values, or `Pending::all` | | A hand-written `impl EncryptFrom for T` | `impl EncryptTarget for T` with `type ContextInput = RequiredContext`, and `DecryptSource` in place of `DecryptInto` | + A typed JSON query, `cipher.query(op::Contains, ..).on(&plan)` and + `op::AtPath(..)`, is the document field's marker query: see "Typed JSON + queries from a field marker", above. + What has no replacement yet: - - **A typed JSON query.** `cipher.query(op::Contains, ..)` and - `op::AtPath(..)` on a plan were the only typed way to query a JSON - document. Until the document's field marker has methods for them, a - JSON query goes through a plan that is data (`dynamic::query_term`, or - `Planned` with a query plan). - **Naming another index for a query** (`.via(index)`). A marker answers through the index its field's type declares. - **A keyset chosen by name on a call** (`.keyset("tenant")`). Load the @@ -222,6 +291,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Implement `op::Answers` and `op::LabelledTarget` for the record by hand to query it. +- **The JSON index answers no typed operator through `Supports`** (#1459). + `Json`'s `Supports` impls for `op::Contains`, `op::ContainedBy` and + `op::AtPath`, `JsonDocument`'s `Answers` and `LabelledTarget` impls, the + `Input` impls of a path and a value, and `json::AtPathTerms` are + removed. Only the removed query chain reached them; a field marker's + JSON queries replace them. The operator types stay: the dynamic `Op` + enum is generated from them. + - **A set of terms holds one index for each operator.** `Encrypted` and a derived `plaintext = T` record now refuse a set that names one index twice, such as `Encrypted<(EqualityTerm, diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index 46fe0ba7af..f11071e26d 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -288,6 +288,22 @@ pub enum Error { #[error(transparent)] #[diagnostic(code(stack_encrypt::other))] Other(Box), + /// The caller's input was refused before any key was requested: a + /// field marker's JSON query whose input has the wrong shape (an + /// `elements` path with no wildcard, a predicate the grammar does not + /// read), or a data plan's source that does not fit the plan, run + /// through `dynamic::Planned`. + /// + /// The refusal is the dynamic interface's own + /// ([`refusal::Error`](crate::refusal::Error)), and it shows through: + /// its message, its code (`stack_encrypt::dynamic_source`, + /// `stack_encrypt::dynamic_predicate` and the rest) and its payload + /// (`reason`, and `field`, `construct` and `at` where it has them) are + /// the ones the dynamic interface reports. It is always the caller's + /// input at fault, never the engine. + #[error(transparent)] + #[diagnostic(transparent)] + Input(Box), /// A [`transcode::Visitor`](crate::target::transcode::Visitor) was handed /// an encrypted output shape its destination does not accept: a scalar /// destination offered a sequence, say. Raised by the visitor's default @@ -461,6 +477,7 @@ impl crate::ErrorPayload for Error { .unwrap_or_default(), #[cfg(not(feature = "http"))] Self::Config(_) => serde_json::Map::new(), + Self::Input(refusal) => refusal.payload(), Self::Aead | Self::JsonNotALeaf | Self::Other(_) diff --git a/packages/stack-encrypt/src/codes.rs b/packages/stack-encrypt/src/codes.rs index 3e5a43fdb6..fc31c56b0d 100644 --- a/packages/stack-encrypt/src/codes.rs +++ b/packages/stack-encrypt/src/codes.rs @@ -68,6 +68,11 @@ mod tests { Error::Config(_) => Error::Config(cause()), Error::Term(_) => Error::Term(TermError::EmptyTermText), Error::Other(_) => Error::Other(cause()), + // The same refusal as the dynamic `Error::Source` row, whose + // code it shows through. + Error::Input(_) => Error::Input(Box::new(crate::refusal::Error::bad_source( + crate::refusal::Reason::FieldMissing, + ))), Error::UnsupportedShape => Error::UnsupportedShape, Error::ContextMismatch { .. } => Error::ContextMismatch { stored: Descriptor::of("users"), @@ -123,6 +128,7 @@ mod tests { TermError::JsonPathNotOrdered => TermError::JsonPathNotOrdered, TermError::JsonOrderingAtPosition => TermError::JsonOrderingAtPosition, TermError::JsonValueNotScalar => TermError::JsonValueNotScalar, + TermError::JsonNumberNotFinite => TermError::JsonNumberNotFinite, ])); errors.extend(boxed(variants![ TermBytesError::WrongEqualityTermLength(_) => { @@ -405,6 +411,15 @@ mod tests { })), json!({ "field": "age" }), ), + ( + Box::new(Error::Input(Box::new(crate::refusal::Error::Predicate { + field: None, + reason: crate::refusal::Reason::JsonPredicateSyntax, + construct: "a path", + at: 3, + }))), + json!({ "reason": "json_predicate_syntax", "construct": "a path", "at": 3 }), + ), ( Box::new(Error::KeyCountMismatch { expected: 2, diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs index f5b3928cd8..124c06f5e1 100644 --- a/packages/stack-encrypt/src/dynamic/json_query.rs +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -204,13 +204,13 @@ use zeroize::Zeroizing; use super::query::{Op, Operand}; use super::{Error, Reason, ValueFrame}; -use crate::json::{ - parse_text, wipe_value, JsonLeafQuery, JsonNode, JsonOptions, JsonPath, JsonPlaintext, - JsonQuery, JsonSubtreeKey, +pub(super) use crate::json::inputs::Under; +use crate::json::inputs::{ + elements_base, json_reason, leaf, member_path, node_path, ordered_path, path, scalar, + selector_path, wildcard_bound, Bounds, Compare, Derived, Input, Parsed, Predicate, }; -use crate::sem::TermError; -use crate::target::{AeadContext, CallerContext, DeclaredContext, Index, Json, Owned}; -use crate::{ContextPiece, DecryptFrom, KeysetCipher, Label, NonEmpty, Pending}; +use crate::json::{parse_text, JsonNode, JsonOptions, JsonPath, JsonPlaintext, JsonSubtreeKey}; +use crate::{DecryptFrom, KeysetCipher, Pending}; /// The SQL type a JSON operand binds as. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -310,20 +310,10 @@ impl<'de> Deserialize<'de> for PathText { } } -/// A document parsed from an input, wiped when it drops. -#[derive(PartialEq)] -pub(super) struct Parsed(pub(super) Value); - -impl Drop for Parsed { - fn drop(&mut self) { - wipe_value(&mut self.0); - } -} - impl JsonText { pub(super) fn parse(&self) -> Result { parse_text(&self.0) - .map(Parsed) + .map(Parsed::of) .map_err(|error| Error::Source { field: None, reason: json_reason(error), @@ -331,16 +321,6 @@ impl JsonText { } } -/// The reason a JSON text refusal is reported with. -pub(super) fn json_reason(error: crate::json::JsonTextError) -> Reason { - use crate::json::JsonTextError as E; - match error { - E::Syntax { .. } => Reason::JsonSyntax, - E::DuplicateKey { .. } => Reason::JsonDuplicateKey, - E::IntegerRange => Reason::JsonIntegerRange, - } -} - #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct DocumentInput { @@ -408,47 +388,6 @@ struct MatchInput { wildcard_bound: Option, } -/// How a predicate of `element_match` compares its field with one value. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum Compare { - Eq, - Lt, - Lte, - Gt, - Gte, -} - -/// A predicate's bounds, as its input gives them. -#[derive(PartialEq)] -pub(super) enum Bounds { - /// One value, compared by the predicate's `op`. - One(Compare, Parsed), - /// `between`: the low bound then the high, both inclusive. - Between(Parsed, Parsed), -} - -/// One predicate of `element_match`, checked: the full path of the -/// element's field, which the field orders, and its bounds. -#[derive(PartialEq)] -pub(super) struct Predicate { - pub(super) path: JsonPath, - pub(super) bounds: Bounds, -} - -/// An operator's input, read from its frame and checked: everything -/// [`derive`] would refuse before it derives anything. `match`'s is its -/// predicate compiled to a tree of the other operators' inputs. -#[derive(PartialEq)] -pub(super) enum Input { - Document(Parsed), - At(JsonPath, Parsed), - Between(JsonPath, Parsed, Parsed), - Path(JsonPath), - Read(JsonPath, u32), - Elements(u32, Vec), - Predicate(Box), -} - fn read_frame Deserialize<'de>>(input: &ValueFrame) -> Result { vitaminc_aead_frame::serde::from_slice(input.risky_ref()).map_err(|_| Error::Source { field: None, @@ -456,59 +395,25 @@ fn read_frame Deserialize<'de>>(input: &ValueFrame) -> Result Result { - let path = - JsonPath::parse(text).map_err(|error| Error::Cipher(TermError::JsonPath(error).into()))?; - if path.segments().len() > crate::json::MAX_DEPTH { - return Err(term(TermError::JsonTooDeep)); - } - Ok(path) -} - -/// A path whose operand is a selector, which reads a position as the -/// wildcard. -fn selector_path(text: &str) -> Result { - let path = path(text)?; - if path.names_a_position() { - return Err(Error::bad_source(Reason::JsonPathPosition)); - } - Ok(path) -} - -/// A path that names one node: no wildcard. -fn node_path(text: &str) -> Result { - let path = path(text)?; - // `is_positional`: every array step names a position, so the path - // has no wildcard and names at most one node. - if path.is_positional() { - Ok(path) - } else { - Err(Error::bad_source(Reason::JsonPathWildcard)) - } -} - -fn term(error: TermError) -> Error { - Error::Cipher(error.into()) -} - -/// Read and check an operator's input against the field's options. +/// Read and check an operator's input against the field's options: the +/// frame read into the input the operator takes ([`Input`]), checked with +/// the rules a field marker's typed JSON queries apply, and in the order a +/// binding has always met them. fn read_input(op: Op, options: &JsonOptions, input: &ValueFrame) -> Result { match op { Op::Contains | Op::ContainedBy => Ok(Input::Document( read_frame::(input)?.json.parse()?, )), Op::EqAt => read_eq_at(input), - Op::LtAt | Op::LteAt | Op::GtAt | Op::GteAt => read_ordering(options, input), + Op::LtAt => read_ordering(options, Compare::Lt, input), + Op::LteAt => read_ordering(options, Compare::Lte, input), + Op::GtAt => read_ordering(options, Compare::Gt, input), + Op::GteAt => read_ordering(options, Compare::Gte, input), Op::BetweenAt => read_between(options, input), - Op::Selector => Ok(Input::Path(selector_path( + Op::Selector => Ok(Input::Selector(selector_path( &read_frame::(input)?.path, )?)), - Op::Children | Op::ChildCount => Ok(Input::Path(node_path( + Op::Children | Op::ChildCount => Ok(Input::Node(node_path( &read_frame::(input)?.path, )?)), Op::Subtree => { @@ -524,15 +429,6 @@ fn read_input(op: Op, options: &JsonOptions, input: &ValueFrame) -> Result) -> Result { - match bound.unwrap_or(JsonSubtreeKey::DEFAULT_BOUND) { - 0 => Err(Error::bad_source(Reason::JsonWildcardBoundZero)), - bound => Ok(bound), - } -} - /// `match`'s input: a jsonpath predicate, its variables as one JSON object /// (the `vars` jsonb `jsonb_path_match` takes, `{}` when absent) and the /// bound its element-scoped leaves are read to. The predicate is compiled @@ -542,29 +438,21 @@ fn read_match(options: &JsonOptions, input: &ValueFrame) -> Result let bound = wildcard_bound(input.wildcard_bound)?; let vars = match &input.vars { Some(vars) => vars.parse()?, - None => Parsed(Value::Object(serde_json::Map::new())), + None => Parsed::of(Value::Object(serde_json::Map::new())), }; if !vars.0.is_object() { return Err(Error::bad_source(Reason::JsonInput)); } let text = std::str::from_utf8(&input.predicate.0) .map_err(|_| Error::bad_source(Reason::JsonInput))?; - Ok(Input::Predicate(Box::new(super::json_predicate::compile( - text, &vars.0, options, bound, - )?))) + Input::predicate(options, text, &vars.0, Some(bound)) } /// `element_match`'s input: an array's elements, by a path with a wildcard /// and no position, and one or more predicates on their fields. fn read_elements(options: &JsonOptions, input: &ValueFrame) -> Result { let elements: ElementsInput = read_frame(input)?; - let base = path(&elements.path)?; - if base.is_positional() { - return Err(Error::bad_source(Reason::JsonPathNoWildcard)); - } - if base.names_a_position() { - return Err(term(TermError::JsonOrderingAtPosition)); - } + let base = elements_base(&elements.path)?; let bound = wildcard_bound(elements.wildcard_bound)?; if elements.predicates.is_empty() { return Err(Error::bad_source(Reason::JsonNoPredicates)); @@ -574,7 +462,7 @@ fn read_elements(options: &JsonOptions, input: &ValueFrame) -> Result, _>>()?; - Ok(Input::Elements(bound, predicates)) + Input::elements(Some(bound), predicates) } /// One predicate of `element_match`: its member's full path below `base`, @@ -607,67 +495,26 @@ fn read_predicate( Ok(Predicate { path, bounds }) } -/// A member's full path: `base` and then the member's steps, read as a -/// path relative to an element (`name`, `address.city`, `["a b"]`). A -/// member names one field of each element, so it has no wildcard of its -/// own, and the full path is an ordering's: no position, and ordered. A -/// member that does not parse is refused at an offset into the member's -/// own text. -fn member_path(options: &JsonOptions, base: &JsonPath, member: &str) -> Result { - let prefix = if member.starts_with('[') { "$" } else { "$." }; - // Sized before it is written, and wiped: the member names keys. - let mut rooted = Zeroizing::new(String::with_capacity(prefix.len() + member.len())); - rooted.push_str(prefix); - rooted.push_str(member); - let relative = JsonPath::parse(&rooted).map_err(|error| { - let error = crate::json::JsonPathError { - at: error.at.saturating_sub(prefix.len()), - ..error - }; - Error::Cipher(TermError::JsonPath(error).into()) - })?; - member_of(options, base, relative) -} - -/// A member's full path, from the steps already parsed: the checks -/// [`member_path`] makes past parsing, which `match` makes of the members -/// of an element-scoped filter. -pub(super) fn member_of( - options: &JsonOptions, - base: &JsonPath, - relative: JsonPath, -) -> Result { - if !relative.is_positional() { - return Err(Error::bad_source(Reason::JsonPathWildcard)); - } - let full = relative - .segments() - .iter() - .fold(base.clone(), |at, segment| at.then(segment.clone())); - if full.segments().len() > crate::json::MAX_DEPTH { - return Err(term(TermError::JsonTooDeep)); - } - ordered_path_of(options, full) -} - /// `eq_at`'s input: a scalar at a path read through a selector. fn read_eq_at(input: &ValueFrame) -> Result { let at: AtInput = read_frame(input)?; let path = selector_path(&at.path)?; let value = at.json.parse()?; - if matches!(value.0, Value::Object(_) | Value::Array(_)) { - return Err(term(TermError::JsonValueNotScalar)); - } - Ok(Input::At(path, value)) + scalar(&value.0)?; + Ok(Input::EqAt(path, value)) } /// An ordering's input: a leaf the index orders at an ordered path. -fn read_ordering(options: &JsonOptions, input: &ValueFrame) -> Result { +fn read_ordering( + options: &JsonOptions, + compare: Compare, + input: &ValueFrame, +) -> Result { let at: AtInput = read_frame(input)?; let path = ordered_path(options, &at.path)?; let value = at.json.parse()?; leaf(&value.0)?; - Ok(Input::At(path, value)) + Ok(Input::Ordering(compare, path, value)) } /// `between_at`'s input: two leaves the index orders at an ordered path. @@ -680,117 +527,23 @@ fn read_between(options: &JsonOptions, input: &ValueFrame) -> Result Result { - ordered_path_of(options, path(text)?) -} - -/// [`ordered_path`] of a path already parsed. -pub(super) fn ordered_path_of(options: &JsonOptions, path: JsonPath) -> Result { - if path.names_a_position() { - Err(term(TermError::JsonOrderingAtPosition)) - } else if options.orders(&path) { - Ok(path) - } else { - Err(term(TermError::JsonPathNotOrdered)) - } -} - -/// A leaf the index orders: a number or a string. -pub(super) fn leaf(value: &Value) -> Result<(), Error> { - match value { - Value::Number(_) | Value::String(_) => Ok(()), - _ => Err(term(TermError::JsonLeafNotOrdered)), - } -} - -/// Check a JSON operator's input without a cipher: what [`derive`] -/// refuses from the input alone (its shape, its paths and values, a path -/// deeper than a document). The limits a subtree read's key hits as it is -/// built, a wildcard expanding too wide or a path too deep for its -/// tokens (`JsonWildcardTooWide`, `JsonTooWide`, `JsonTooDeep`), are -/// [`derive`]'s, after the keyset is resolved. +/// Check a JSON operator's input without a cipher: what [`derive`] refuses +/// from the input alone (its shape, its paths and values, a path deeper +/// than a document). The limits a subtree read's key hits as it is built, +/// a wildcard expanding too wide or a path too deep for its tokens +/// (`JsonWildcardTooWide`, `JsonTooWide`, `JsonTooDeep`), are [`derive`]'s, +/// after the keyset is resolved. pub(super) fn check(op: Op, options: &JsonOptions, input: &ValueFrame) -> Result<(), Error> { read_input(op, options, input).map(drop) } -/// Where a document field's queries and reads run: under the field's -/// label, then the scopes and the bindings of its plan, as the document -/// was sealed. -/// -/// For example, take the field `profile` of the table `users`, in a plan -/// whose caller added the tenant 7 as a scope and the row 9 as a binding. -/// A selector, a read key and every other term derive under -/// `users/profile`, then 7: the query context. An entry that a read -/// returns opens under `users/profile`, then 7, then 9: the stored -/// context. -#[derive(Clone)] -pub(super) struct Under<'p> { - label: &'p Label, - /// The plan's scopes, then its bindings. - parts: DeclaredContext, -} - -impl<'p> Under<'p> { - /// The field with the label `label`, in a plan whose scopes and - /// bindings are `parts`. - pub(super) fn new(label: &'p Label, parts: DeclaredContext) -> Self { - Self { label, parts } - } - - fn column(&self) -> NonEmpty